diff options
author | Martin Liska <mliska@suse.cz> | 2022-08-30 11:45:34 +0200 |
---|---|---|
committer | Martin Liska <mliska@suse.cz> | 2022-08-30 12:53:50 +0200 |
commit | 600413c4f3d70392285192fb99634bcbeb97f83f (patch) | |
tree | 0586f1cc2feaa4f5a3d632926b08bde261c39786 | |
parent | bdd3547ae4279c14a9db883719c9648ed09dc18a (diff) | |
download | gcc-600413c4f3d70392285192fb99634bcbeb97f83f.tar.gz |
libsanitizer: merge from master (84a71d5259c2682403cdbd8710592410a2f128ab)
121 files changed, 2132 insertions, 883 deletions
diff --git a/libsanitizer/MERGE b/libsanitizer/MERGE index 691c82b0003..3ca670c1184 100644 --- a/libsanitizer/MERGE +++ b/libsanitizer/MERGE @@ -1,4 +1,4 @@ -75f9e83ace52773af65dcebca543005ec8a2705d +84a71d5259c2682403cdbd8710592410a2f128ab The first line of this file holds the git revision number of the last merge done from the master library sources. diff --git a/libsanitizer/asan/asan_errors.cpp b/libsanitizer/asan/asan_errors.cpp index a22bf130d82..10f7c17991d 100644 --- a/libsanitizer/asan/asan_errors.cpp +++ b/libsanitizer/asan/asan_errors.cpp @@ -279,9 +279,7 @@ void ErrorRssLimitExceeded::Print() { void ErrorOutOfMemory::Print() { Decorator d; Printf("%s", d.Error()); - Report( - "ERROR: AddressSanitizer: allocator is out of memory trying to allocate " - "0x%zx bytes\n", requested_size); + ERROR_OOM("allocator is trying to allocate 0x%zx bytes\n", requested_size); Printf("%s", d.Default()); stack->Print(); PrintHintAllocatorCannotReturnNull(); diff --git a/libsanitizer/asan/asan_flags.cpp b/libsanitizer/asan/asan_flags.cpp index 9ea899f84b4..23989843323 100644 --- a/libsanitizer/asan/asan_flags.cpp +++ b/libsanitizer/asan/asan_flags.cpp @@ -87,7 +87,7 @@ void InitializeFlags() { RegisterCommonFlags(&ubsan_parser); #endif - if (SANITIZER_MAC) { + if (SANITIZER_APPLE) { // Support macOS MallocScribble and MallocPreScribble: // <https://developer.apple.com/library/content/documentation/Performance/ // Conceptual/ManagingMemory/Articles/MallocDebug.html> diff --git a/libsanitizer/asan/asan_flags.inc b/libsanitizer/asan/asan_flags.inc index 314ed193535..fad1577d912 100644 --- a/libsanitizer/asan/asan_flags.inc +++ b/libsanitizer/asan/asan_flags.inc @@ -83,6 +83,10 @@ ASAN_FLAG( int, sleep_after_init, 0, "Number of seconds to sleep after AddressSanitizer is initialized. " "Useful for debugging purposes (e.g. when one needs to attach gdb).") +ASAN_FLAG( + int, sleep_before_init, 0, + "Number of seconds to sleep before AddressSanitizer starts initializing. " + "Useful for debugging purposes (e.g. when one needs to attach gdb).") ASAN_FLAG(bool, check_malloc_usable_size, true, "Allows the users to work around the bug in Nvidia drivers prior to " "295.*.") @@ -118,7 +122,7 @@ ASAN_FLAG(bool, poison_array_cookie, true, // https://github.com/google/sanitizers/issues/309 // TODO(glider,timurrrr): Fix known issues and enable this back. ASAN_FLAG(bool, alloc_dealloc_mismatch, - !SANITIZER_MAC && !SANITIZER_WINDOWS && !SANITIZER_ANDROID, + !SANITIZER_APPLE && !SANITIZER_WINDOWS && !SANITIZER_ANDROID, "Report errors on malloc/delete, new/free, new/delete[], etc.") ASAN_FLAG(bool, new_delete_type_mismatch, true, diff --git a/libsanitizer/asan/asan_globals.cpp b/libsanitizer/asan/asan_globals.cpp index 326a49798f0..b780128c9ad 100644 --- a/libsanitizer/asan/asan_globals.cpp +++ b/libsanitizer/asan/asan_globals.cpp @@ -86,10 +86,11 @@ static void ReportGlobal(const Global &g, const char *prefix) { "odr_indicator=%p\n", prefix, (void *)&g, (void *)g.beg, g.size, g.size_with_redzone, g.name, g.module_name, g.has_dynamic_init, (void *)g.odr_indicator); - if (g.location) { - Report(" location (%p): name=%s[%p], %d %d\n", (void *)g.location, - g.location->filename, (void *)g.location->filename, - g.location->line_no, g.location->column_no); + + DataInfo info; + Symbolizer::GetOrInit()->SymbolizeData(g.beg, &info); + if (info.line != 0) { + Report(" location: name=%s, %d\n", info.file, static_cast<int>(info.line)); } } @@ -153,6 +154,23 @@ static void CheckODRViolationViaIndicator(const Global *g) { } } +// Check ODR violation for given global G by checking if it's already poisoned. +// We use this method in case compiler doesn't use private aliases for global +// variables. +static void CheckODRViolationViaPoisoning(const Global *g) { + if (__asan_region_is_poisoned(g->beg, g->size_with_redzone)) { + // This check may not be enough: if the first global is much larger + // the entire redzone of the second global may be within the first global. + for (ListOfGlobals *l = list_of_all_globals; l; l = l->next) { + if (g->beg == l->g->beg && + (flags()->detect_odr_violation >= 2 || g->size != l->g->size) && + !IsODRViolationSuppressed(g->name)) + ReportODRViolation(g, FindRegistrationSite(g), + l->g, FindRegistrationSite(l->g)); + } + } +} + // Clang provides two different ways for global variables protection: // it can poison the global itself or its private alias. In former // case we may poison same symbol multiple times, that can help us to @@ -198,6 +216,8 @@ static void RegisterGlobal(const Global *g) { // where two globals with the same name are defined in different modules. if (UseODRIndicator(g)) CheckODRViolationViaIndicator(g); + else + CheckODRViolationViaPoisoning(g); } if (CanPoisonMemory()) PoisonRedZones(*g); @@ -276,19 +296,15 @@ void PrintGlobalNameIfASCII(InternalScopedString *str, const __asan_global &g) { (char *)g.beg); } -static const char *GlobalFilename(const __asan_global &g) { - const char *res = g.module_name; - // Prefer the filename from source location, if is available. - if (g.location) res = g.location->filename; - CHECK(res); - return res; -} - void PrintGlobalLocation(InternalScopedString *str, const __asan_global &g) { - str->append("%s", GlobalFilename(g)); - if (!g.location) return; - if (g.location->line_no) str->append(":%d", g.location->line_no); - if (g.location->column_no) str->append(":%d", g.location->column_no); + DataInfo info; + Symbolizer::GetOrInit()->SymbolizeData(g.beg, &info); + + if (info.line != 0) { + str->append("%s:%d", info.file, static_cast<int>(info.line)); + } else { + str->append("%s", g.module_name); + } } } // namespace __asan diff --git a/libsanitizer/asan/asan_interceptors.cpp b/libsanitizer/asan/asan_interceptors.cpp index 2ff314a5a9c..817008253fc 100644 --- a/libsanitizer/asan/asan_interceptors.cpp +++ b/libsanitizer/asan/asan_interceptors.cpp @@ -103,7 +103,7 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) do { \ if (asan_init_is_running) \ return REAL(func)(__VA_ARGS__); \ - if (SANITIZER_MAC && UNLIKELY(!asan_inited)) \ + if (SANITIZER_APPLE && UNLIKELY(!asan_inited)) \ return REAL(func)(__VA_ARGS__); \ ENSURE_ASAN_INITED(); \ } while (false) @@ -243,15 +243,26 @@ DEFINE_REAL_PTHREAD_FUNCTIONS #if ASAN_INTERCEPT_SWAPCONTEXT static void ClearShadowMemoryForContextStack(uptr stack, uptr ssize) { + // Only clear if we know the stack. This should be true only for contexts + // created with makecontext(). + if (!ssize) + return; // Align to page size. uptr PageSize = GetPageSizeCached(); - uptr bottom = stack & ~(PageSize - 1); + uptr bottom = RoundDownTo(stack, PageSize); + if (!AddrIsInMem(bottom)) + return; ssize += stack - bottom; ssize = RoundUpTo(ssize, PageSize); - static const uptr kMaxSaneContextStackSize = 1 << 22; // 4 Mb - if (AddrIsInMem(bottom) && ssize && ssize <= kMaxSaneContextStackSize) { - PoisonShadow(bottom, ssize, 0); - } + PoisonShadow(bottom, ssize, 0); +} + +INTERCEPTOR(int, getcontext, struct ucontext_t *ucp) { + // API does not requires to have ucp clean, and sets only part of fields. We + // use ucp->uc_stack to unpoison new stack. We prefer to have zeroes then + // uninitialized bytes. + ResetContextStack(ucp); + return REAL(getcontext)(ucp); } INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp, @@ -267,15 +278,18 @@ INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp, uptr stack, ssize; ReadContextStack(ucp, &stack, &ssize); ClearShadowMemoryForContextStack(stack, ssize); -#if __has_attribute(__indirect_return__) && \ - (defined(__x86_64__) || defined(__i386__)) + + // See getcontext interceptor. + ResetContextStack(oucp); + +# if __has_attribute(__indirect_return__) && \ + (defined(__x86_64__) || defined(__i386__)) int (*real_swapcontext)(struct ucontext_t *, struct ucontext_t *) - __attribute__((__indirect_return__)) - = REAL(swapcontext); + __attribute__((__indirect_return__)) = REAL(swapcontext); int res = real_swapcontext(oucp, ucp); -#else +# else int res = REAL(swapcontext)(oucp, ucp); -#endif +# endif // swapcontext technically does not return, but program may swap context to // "oucp" later, that would look as if swapcontext() returned 0. // We need to clear shadow for ucp once again, as it may be in arbitrary @@ -355,7 +369,7 @@ INTERCEPTOR(_Unwind_Reason_Code, _Unwind_SjLj_RaiseException, INTERCEPTOR(char*, index, const char *string, int c) ALIAS(WRAPPER_NAME(strchr)); # else -# if SANITIZER_MAC +# if SANITIZER_APPLE DECLARE_REAL(char*, index, const char *string, int c) OVERRIDE_FUNCTION(index, strchr); # else @@ -409,7 +423,7 @@ INTERCEPTOR(char*, strncat, char *to, const char *from, uptr size) { INTERCEPTOR(char *, strcpy, char *to, const char *from) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, strcpy); -#if SANITIZER_MAC +#if SANITIZER_APPLE if (UNLIKELY(!asan_inited)) return REAL(strcpy)(to, from); #endif @@ -489,7 +503,7 @@ INTERCEPTOR(long, strtol, const char *nptr, char **endptr, int base) { INTERCEPTOR(int, atoi, const char *nptr) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, atoi); -#if SANITIZER_MAC +#if SANITIZER_APPLE if (UNLIKELY(!asan_inited)) return REAL(atoi)(nptr); #endif ENSURE_ASAN_INITED(); @@ -510,7 +524,7 @@ INTERCEPTOR(int, atoi, const char *nptr) { INTERCEPTOR(long, atol, const char *nptr) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, atol); -#if SANITIZER_MAC +#if SANITIZER_APPLE if (UNLIKELY(!asan_inited)) return REAL(atol)(nptr); #endif ENSURE_ASAN_INITED(); @@ -563,7 +577,7 @@ static void AtCxaAtexit(void *unused) { #if ASAN_INTERCEPT___CXA_ATEXIT INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, void *dso_handle) { -#if SANITIZER_MAC +#if SANITIZER_APPLE if (UNLIKELY(!asan_inited)) return REAL(__cxa_atexit)(func, arg, dso_handle); #endif ENSURE_ASAN_INITED(); @@ -645,6 +659,7 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(longjmp); #if ASAN_INTERCEPT_SWAPCONTEXT + ASAN_INTERCEPT_FUNC(getcontext); ASAN_INTERCEPT_FUNC(swapcontext); #endif #if ASAN_INTERCEPT__LONGJMP diff --git a/libsanitizer/asan/asan_interceptors.h b/libsanitizer/asan/asan_interceptors.h index 105c672cc24..35727a96497 100644 --- a/libsanitizer/asan/asan_interceptors.h +++ b/libsanitizer/asan/asan_interceptors.h @@ -81,12 +81,7 @@ void InitializePlatformInterceptors(); #if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && !SANITIZER_SOLARIS && \ !SANITIZER_NETBSD # define ASAN_INTERCEPT___CXA_THROW 1 -# if ! defined(ASAN_HAS_CXA_RETHROW_PRIMARY_EXCEPTION) \ - || ASAN_HAS_CXA_RETHROW_PRIMARY_EXCEPTION -# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1 -# else -# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 0 -# endif +# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1 # if defined(_GLIBCXX_SJLJ_EXCEPTIONS) || (SANITIZER_IOS && defined(__arm__)) # define ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION 1 # else @@ -138,7 +133,7 @@ DECLARE_REAL(char*, strncpy, char *to, const char *from, uptr size) DECLARE_REAL(uptr, strnlen, const char *s, uptr maxlen) DECLARE_REAL(char*, strstr, const char *s1, const char *s2) -# if !SANITIZER_MAC +# if !SANITIZER_APPLE # define ASAN_INTERCEPT_FUNC(name) \ do { \ if (!INTERCEPT_FUNCTION(name)) \ @@ -161,7 +156,7 @@ DECLARE_REAL(char*, strstr, const char *s1, const char *s2) # else // OS X interceptors don't need to be initialized with INTERCEPT_FUNCTION. # define ASAN_INTERCEPT_FUNC(name) -# endif // SANITIZER_MAC +# endif // SANITIZER_APPLE #endif // !SANITIZER_FUCHSIA diff --git a/libsanitizer/asan/asan_interceptors_memintrinsics.h b/libsanitizer/asan/asan_interceptors_memintrinsics.h index 632f0515a9e..bbc5390ceaa 100644 --- a/libsanitizer/asan/asan_interceptors_memintrinsics.h +++ b/libsanitizer/asan/asan_interceptors_memintrinsics.h @@ -18,26 +18,29 @@ #include "asan_mapping.h" #include "interception/interception.h" -DECLARE_REAL(void*, memcpy, void *to, const void *from, uptr size) -DECLARE_REAL(void*, memset, void *block, int c, uptr size) +DECLARE_REAL(void *, memcpy, void *to, const void *from, uptr size) +DECLARE_REAL(void *, memset, void *block, int c, uptr size) namespace __asan { // Return true if we can quickly decide that the region is unpoisoned. // We assume that a redzone is at least 16 bytes. static inline bool QuickCheckForUnpoisonedRegion(uptr beg, uptr size) { - if (size == 0) return true; - if (size <= 32) - return !AddressIsPoisoned(beg) && - !AddressIsPoisoned(beg + size - 1) && - !AddressIsPoisoned(beg + size / 2); - if (size <= 64) - return !AddressIsPoisoned(beg) && - !AddressIsPoisoned(beg + size / 4) && - !AddressIsPoisoned(beg + size - 1) && - !AddressIsPoisoned(beg + 3 * size / 4) && - !AddressIsPoisoned(beg + size / 2); - return false; + if (UNLIKELY(size == 0 || size > sizeof(uptr) * ASAN_SHADOW_GRANULARITY)) + return !size; + + uptr last = beg + size - 1; + uptr shadow_first = MEM_TO_SHADOW(beg); + uptr shadow_last = MEM_TO_SHADOW(last); + uptr uptr_first = RoundDownTo(shadow_first, sizeof(uptr)); + uptr uptr_last = RoundDownTo(shadow_last, sizeof(uptr)); + if (LIKELY(((*reinterpret_cast<const uptr *>(uptr_first) | + *reinterpret_cast<const uptr *>(uptr_last)) == 0))) + return true; + u8 shadow = AddressIsPoisoned(last); + for (; shadow_first < shadow_last; ++shadow_first) + shadow |= *((u8 *)shadow_first); + return !shadow; } struct AsanInterceptorContext { @@ -49,75 +52,68 @@ struct AsanInterceptorContext { // that no extra frames are created, and stack trace contains // relevant information only. // We check all shadow bytes. -#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) do { \ - uptr __offset = (uptr)(offset); \ - uptr __size = (uptr)(size); \ - uptr __bad = 0; \ - if (__offset > __offset + __size) { \ - GET_STACK_TRACE_FATAL_HERE; \ - ReportStringFunctionSizeOverflow(__offset, __size, &stack); \ - } \ - if (!QuickCheckForUnpoisonedRegion(__offset, __size) && \ - (__bad = __asan_region_is_poisoned(__offset, __size))) { \ - AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \ - bool suppressed = false; \ - if (_ctx) { \ - suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \ - if (!suppressed && HaveStackTraceBasedSuppressions()) { \ - GET_STACK_TRACE_FATAL_HERE; \ - suppressed = IsStackTraceSuppressed(&stack); \ - } \ - } \ - if (!suppressed) { \ - GET_CURRENT_PC_BP_SP; \ - ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\ - } \ - } \ +#define ACCESS_MEMORY_RANGE(ctx, offset, size, isWrite) \ + do { \ + uptr __offset = (uptr)(offset); \ + uptr __size = (uptr)(size); \ + uptr __bad = 0; \ + if (UNLIKELY(__offset > __offset + __size)) { \ + GET_STACK_TRACE_FATAL_HERE; \ + ReportStringFunctionSizeOverflow(__offset, __size, &stack); \ + } \ + if (UNLIKELY(!QuickCheckForUnpoisonedRegion(__offset, __size)) && \ + (__bad = __asan_region_is_poisoned(__offset, __size))) { \ + AsanInterceptorContext *_ctx = (AsanInterceptorContext *)ctx; \ + bool suppressed = false; \ + if (_ctx) { \ + suppressed = IsInterceptorSuppressed(_ctx->interceptor_name); \ + if (!suppressed && HaveStackTraceBasedSuppressions()) { \ + GET_STACK_TRACE_FATAL_HERE; \ + suppressed = IsStackTraceSuppressed(&stack); \ + } \ + } \ + if (!suppressed) { \ + GET_CURRENT_PC_BP_SP; \ + ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false); \ + } \ + } \ } while (0) // memcpy is called during __asan_init() from the internals of printf(...). // We do not treat memcpy with to==from as a bug. // See http://llvm.org/bugs/show_bug.cgi?id=11763. -#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memcpy(to, from, size); \ - if (asan_init_is_running) { \ - return REAL(memcpy)(to, from, size); \ - } \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - if (to != from) { \ - CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ - } \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } \ - return REAL(memcpy)(to, from, size); \ +#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + if (LIKELY(to != from)) { \ + CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ + } \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } else if (UNLIKELY(!asan_inited)) { \ + return internal_memcpy(to, from, size); \ + } \ + return REAL(memcpy)(to, from, size); \ } while (0) // memset is called inside Printf. -#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memset(block, c, size); \ - if (asan_init_is_running) { \ - return REAL(memset)(block, c, size); \ - } \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - ASAN_WRITE_RANGE(ctx, block, size); \ - } \ - return REAL(memset)(block, c, size); \ +#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + ASAN_WRITE_RANGE(ctx, block, size); \ + } else if (UNLIKELY(!asan_inited)) { \ + return internal_memset(block, c, size); \ + } \ + return REAL(memset)(block, c, size); \ } while (0) -#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ - do { \ - if (UNLIKELY(!asan_inited)) return internal_memmove(to, from, size); \ - ENSURE_ASAN_INITED(); \ - if (flags()->replace_intrin) { \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } \ - return internal_memmove(to, from, size); \ +#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } \ + return internal_memmove(to, from, size); \ } while (0) #define ASAN_READ_RANGE(ctx, offset, size) \ @@ -136,7 +132,7 @@ static inline bool RangesOverlap(const char *offset1, uptr length1, do { \ const char *offset1 = (const char *)_offset1; \ const char *offset2 = (const char *)_offset2; \ - if (RangesOverlap(offset1, length1, offset2, length2)) { \ + if (UNLIKELY(RangesOverlap(offset1, length1, offset2, length2))) { \ GET_STACK_TRACE_FATAL_HERE; \ bool suppressed = IsInterceptorSuppressed(name); \ if (!suppressed && HaveStackTraceBasedSuppressions()) { \ diff --git a/libsanitizer/asan/asan_interceptors_vfork.S b/libsanitizer/asan/asan_interceptors_vfork.S index 3ae5503e83c..ec29adc7b13 100644 --- a/libsanitizer/asan/asan_interceptors_vfork.S +++ b/libsanitizer/asan/asan_interceptors_vfork.S @@ -6,6 +6,7 @@ #include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S" #include "sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S" #include "sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S" #include "sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S" #include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S" #endif diff --git a/libsanitizer/asan/asan_interface_internal.h b/libsanitizer/asan/asan_interface_internal.h index 3e6e6602887..b0802a89ddb 100644 --- a/libsanitizer/asan/asan_interface_internal.h +++ b/libsanitizer/asan/asan_interface_internal.h @@ -53,8 +53,9 @@ extern "C" { const char *module_name; // Module name as a C string. This pointer is a // unique identifier of a module. uptr has_dynamic_init; // Non-zero if the global has dynamic initializer. - __asan_global_source_location *location; // Source location of a global, - // or NULL if it is unknown. + uptr windows_padding; // TODO: Figure out how to remove this padding + // that's simply here to make the MSVC incremental + // linker happy... uptr odr_indicator; // The address of the ODR indicator symbol. }; diff --git a/libsanitizer/asan/asan_internal.h b/libsanitizer/asan/asan_internal.h index 7468f126d37..959fdec2604 100644 --- a/libsanitizer/asan/asan_internal.h +++ b/libsanitizer/asan/asan_internal.h @@ -106,6 +106,7 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle); void AsanOnDeadlySignal(int, void *siginfo, void *context); void ReadContextStack(void *context, uptr *stack, uptr *ssize); +void ResetContextStack(void *context); void StopInitOrderChecking(); // Wrapper for TLS/TSD. @@ -132,6 +133,7 @@ void InstallAtExitCheckLeaks(); extern int asan_inited; // Used to avoid infinite recursion in __asan_init(). extern bool asan_init_is_running; +extern bool replace_intrin_cached; extern void (*death_callback)(void); // These magic values are written to shadow for better error // reporting. diff --git a/libsanitizer/asan/asan_linux.cpp b/libsanitizer/asan/asan_linux.cpp index defd81bc19e..89450fc120a 100644 --- a/libsanitizer/asan/asan_linux.cpp +++ b/libsanitizer/asan/asan_linux.cpp @@ -214,11 +214,19 @@ void ReadContextStack(void *context, uptr *stack, uptr *ssize) { *stack = (uptr)ucp->uc_stack.ss_sp; *ssize = ucp->uc_stack.ss_size; } -#else + +void ResetContextStack(void *context) { + ucontext_t *ucp = (ucontext_t *)context; + ucp->uc_stack.ss_sp = nullptr; + ucp->uc_stack.ss_size = 0; +} +# else void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } -#endif + +void ResetContextStack(void *context) { UNIMPLEMENTED(); } +# endif void *AsanDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); diff --git a/libsanitizer/asan/asan_mac.cpp b/libsanitizer/asan/asan_mac.cpp index 9161f728d44..a2d5c31a3f7 100644 --- a/libsanitizer/asan/asan_mac.cpp +++ b/libsanitizer/asan/asan_mac.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "asan_interceptors.h" #include "asan_internal.h" @@ -99,6 +99,8 @@ void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } +void ResetContextStack(void *context) { UNIMPLEMENTED(); } + // Support for the following functions from libdispatch on Mac OS: // dispatch_async_f() // dispatch_async() @@ -296,4 +298,4 @@ INTERCEPTOR(void, dispatch_source_set_event_handler, } #endif -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/asan/asan_malloc_mac.cpp b/libsanitizer/asan/asan_malloc_mac.cpp index e8484685dae..924d1f12640 100644 --- a/libsanitizer/asan/asan_malloc_mac.cpp +++ b/libsanitizer/asan/asan_malloc_mac.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "asan_interceptors.h" #include "asan_report.h" diff --git a/libsanitizer/asan/asan_mapping.h b/libsanitizer/asan/asan_mapping.h index 1b6669e1270..aeadb9d94eb 100644 --- a/libsanitizer/asan/asan_mapping.h +++ b/libsanitizer/asan/asan_mapping.h @@ -114,6 +114,13 @@ // || `[0x0080000000000, 0x008ffffffffff]` || LowShadow || // || `[0x0000000000000, 0x007ffffffffff]` || LowMem || // +// Default Linux/LoongArch64 (47-bit VMA) mapping: +// || `[0x500000000000, 0x7fffffffffff]` || HighMem || +// || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow || +// || `[0x480000000000, 0x49ffffffffff]` || ShadowGap || +// || `[0x400000000000, 0x47ffffffffff]` || LowShadow || +// || `[0x000000000000, 0x3fffffffffff]` || LowMem || +// // Shadow mapping on FreeBSD/x86-64 with SHADOW_OFFSET == 0x400000000000: // || `[0x500000000000, 0x7fffffffffff]` || HighMem || // || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow || @@ -174,26 +181,30 @@ #else # if SANITIZER_IOS # define ASAN_SHADOW_OFFSET_DYNAMIC -# elif SANITIZER_MAC && defined(__aarch64__) +# elif SANITIZER_APPLE && defined(__aarch64__) # define ASAN_SHADOW_OFFSET_DYNAMIC +# elif SANITIZER_FREEBSD && defined(__aarch64__) +# define ASAN_SHADOW_OFFSET_CONST 0x0000800000000000 # elif SANITIZER_RISCV64 # define ASAN_SHADOW_OFFSET_CONST 0x0000000d55550000 # elif defined(__aarch64__) # define ASAN_SHADOW_OFFSET_CONST 0x0000001000000000 # elif defined(__powerpc64__) -# define ASAN_SHADOW_OFFSET_CONST 0x0000020000000000 +# define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000 # elif defined(__s390x__) # define ASAN_SHADOW_OFFSET_CONST 0x0010000000000000 # elif SANITIZER_FREEBSD # define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000 # elif SANITIZER_NETBSD # define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000 -# elif SANITIZER_MAC +# elif SANITIZER_APPLE # define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000 # elif defined(__mips64) # define ASAN_SHADOW_OFFSET_CONST 0x0000002000000000 # elif defined(__sparc__) # define ASAN_SHADOW_OFFSET_CONST 0x0000080000000000 +# elif SANITIZER_LOONGARCH64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000 # elif SANITIZER_WINDOWS64 # define ASAN_SHADOW_OFFSET_DYNAMIC # else diff --git a/libsanitizer/asan/asan_new_delete.cpp b/libsanitizer/asan/asan_new_delete.cpp index da446072de1..17280129c75 100644 --- a/libsanitizer/asan/asan_new_delete.cpp +++ b/libsanitizer/asan/asan_new_delete.cpp @@ -89,7 +89,7 @@ enum class align_val_t: size_t {}; // delete. // To make sure that C++ allocation/deallocation operators are overridden on // OS X we need to intercept them using their mangled names. -#if !SANITIZER_MAC +#if !SANITIZER_APPLE CXX_OPERATOR_ATTRIBUTE void *operator new(size_t size) { OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); } @@ -115,7 +115,7 @@ CXX_OPERATOR_ATTRIBUTE void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&) { OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/); } -#else // SANITIZER_MAC +#else // SANITIZER_APPLE INTERCEPTOR(void *, _Znwm, size_t size) { OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); } @@ -128,7 +128,7 @@ INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) { INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); } -#endif // !SANITIZER_MAC +#endif // !SANITIZER_APPLE #define OPERATOR_DELETE_BODY(type) \ GET_STACK_TRACE_FREE; \ @@ -146,7 +146,7 @@ INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) { GET_STACK_TRACE_FREE; \ asan_delete(ptr, size, static_cast<uptr>(align), &stack, type); -#if !SANITIZER_MAC +#if !SANITIZER_APPLE CXX_OPERATOR_ATTRIBUTE void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY(FROM_NEW); } @@ -184,7 +184,7 @@ CXX_OPERATOR_ATTRIBUTE void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT { OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); } -#else // SANITIZER_MAC +#else // SANITIZER_APPLE INTERCEPTOR(void, _ZdlPv, void *ptr) { OPERATOR_DELETE_BODY(FROM_NEW); } INTERCEPTOR(void, _ZdaPv, void *ptr) @@ -193,4 +193,4 @@ INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY(FROM_NEW); } INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY(FROM_NEW_BR); } -#endif // !SANITIZER_MAC +#endif // !SANITIZER_APPLE diff --git a/libsanitizer/asan/asan_rtl.cpp b/libsanitizer/asan/asan_rtl.cpp index 3a5261474b2..88f66de5669 100644 --- a/libsanitizer/asan/asan_rtl.cpp +++ b/libsanitizer/asan/asan_rtl.cpp @@ -51,10 +51,9 @@ static void AsanDie() { } if (common_flags()->print_module_map >= 1) DumpProcessMap(); - if (flags()->sleep_before_dying) { - Report("Sleeping for %d second(s)\n", flags()->sleep_before_dying); - SleepForSeconds(flags()->sleep_before_dying); - } + + WaitForDebugger(flags()->sleep_before_dying, "before dying"); + if (flags()->unmap_shadow_on_exit) { if (kMidMemBeg) { UnmapOrDie((void*)kLowShadowBeg, kMidMemBeg - kLowShadowBeg); @@ -74,6 +73,7 @@ static void CheckUnwind() { // -------------------------- Globals --------------------- {{{1 int asan_inited; bool asan_init_is_running; +bool replace_intrin_cached; #if !ASAN_FIXED_MAPPING uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; @@ -386,6 +386,8 @@ static void AsanInitInternal() { // initialization steps look at flags(). InitializeFlags(); + WaitForDebugger(flags()->sleep_before_init, "before init"); + // Stop performing init at this point if we are being loaded via // dlopen() and the platform supports it. if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) { @@ -420,9 +422,6 @@ static void AsanInitInternal() { __sanitizer::InitializePlatformEarly(); - // Re-exec ourselves if we need to set additional env or command line args. - MaybeReexec(); - // Setup internal allocator callback. SetLowLevelAllocateMinAlignment(ASAN_SHADOW_GRANULARITY); SetLowLevelAllocateCallback(OnLowLevelAllocate); @@ -453,6 +452,7 @@ static void AsanInitInternal() { // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited // should be set to 1 prior to initializing the threads. + replace_intrin_cached = flags()->replace_intrin; asan_inited = 1; asan_init_is_running = false; @@ -497,10 +497,7 @@ static void AsanInitInternal() { VReport(1, "AddressSanitizer Init done\n"); - if (flags()->sleep_after_init) { - Report("Sleeping for %d second(s)\n", flags()->sleep_after_init); - SleepForSeconds(flags()->sleep_after_init); - } + WaitForDebugger(flags()->sleep_after_init, "after init"); } // Initialize as requested from some part of ASan runtime library (interceptors, diff --git a/libsanitizer/asan/asan_win.cpp b/libsanitizer/asan/asan_win.cpp index 81958038fb1..f11df0613d1 100644 --- a/libsanitizer/asan/asan_win.cpp +++ b/libsanitizer/asan/asan_win.cpp @@ -267,6 +267,8 @@ void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } +void ResetContextStack(void *context) { UNIMPLEMENTED(); } + void AsanOnDeadlySignal(int, void *siginfo, void *context) { UNIMPLEMENTED(); } bool PlatformUnpoisonStacks() { return false; } diff --git a/libsanitizer/hwasan/hwasan.cpp b/libsanitizer/hwasan/hwasan.cpp index f8725a17343..bb946c2ffe0 100644 --- a/libsanitizer/hwasan/hwasan.cpp +++ b/libsanitizer/hwasan/hwasan.cpp @@ -218,8 +218,8 @@ void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc, registers_frame); } -void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame, - size_t outsize) { +void HwasanTagMismatch(uptr addr, uptr pc, uptr frame, uptr access_info, + uptr *registers_frame, size_t outsize) { __hwasan::AccessInfo ai; ai.is_store = access_info & 0x10; ai.is_load = !ai.is_store; @@ -230,9 +230,7 @@ void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame, else ai.size = 1 << (access_info & 0xf); - HandleTagMismatch(ai, (uptr)__builtin_return_address(0), - (uptr)__builtin_frame_address(0), nullptr, registers_frame); - __builtin_unreachable(); + HandleTagMismatch(ai, pc, frame, nullptr, registers_frame); } Thread *GetCurrentThread() { @@ -576,6 +574,12 @@ u8 __hwasan_generate_tag() { return t->GenerateRandomTag(); } +void __hwasan_add_frame_record(u64 frame_record_info) { + Thread *t = GetCurrentThread(); + if (t) + t->stack_allocations()->push(frame_record_info); +} + #if !SANITIZER_SUPPORTS_WEAK_HOOKS extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE @@ -594,7 +598,9 @@ void __sanitizer_print_stack_trace() { // rest of the mismatch handling code (C++). void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, size_t outsize) { - __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize); + __hwasan::HwasanTagMismatch(addr, (uptr)__builtin_return_address(0), + (uptr)__builtin_frame_address(0), access_info, + registers_frame, outsize); } } // extern "C" diff --git a/libsanitizer/hwasan/hwasan.h b/libsanitizer/hwasan/hwasan.h index 3cc2fc40b5f..ef4055a50ef 100644 --- a/libsanitizer/hwasan/hwasan.h +++ b/libsanitizer/hwasan/hwasan.h @@ -167,8 +167,8 @@ void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc, // This dispatches to HandleTagMismatch but sets up the AccessInfo, program // counter, and frame pointer. -void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame, - size_t outsize); +void HwasanTagMismatch(uptr addr, uptr pc, uptr frame, uptr access_info, + uptr *registers_frame, size_t outsize); } // namespace __hwasan @@ -181,6 +181,13 @@ typedef unsigned long __hw_sigset_t; constexpr size_t kHwRegisterBufSize = 22; # elif defined(__x86_64__) constexpr size_t kHwRegisterBufSize = 8; +# elif SANITIZER_RISCV64 +// saving PC, 12 int regs, sp, 12 fp regs +# ifndef __riscv_float_abi_soft +constexpr size_t kHwRegisterBufSize = 1 + 12 + 1 + 12; +# else +constexpr size_t kHwRegisterBufSize = 1 + 12 + 1; +# endif # endif typedef unsigned long long __hw_register_buf[kHwRegisterBufSize]; struct __hw_jmp_buf_struct { diff --git a/libsanitizer/hwasan/hwasan_allocator.h b/libsanitizer/hwasan/hwasan_allocator.h index 35c3d6b4bf4..bae53b56559 100644 --- a/libsanitizer/hwasan/hwasan_allocator.h +++ b/libsanitizer/hwasan/hwasan_allocator.h @@ -24,8 +24,8 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_ring_buffer.h" -#if !defined(__aarch64__) && !defined(__x86_64__) -#error Unsupported platform +#if !defined(__aarch64__) && !defined(__x86_64__) && !(SANITIZER_RISCV64) +# error Unsupported platform #endif namespace __hwasan { diff --git a/libsanitizer/hwasan/hwasan_checks.h b/libsanitizer/hwasan/hwasan_checks.h index ab543ea88be..b0b37d7a2e2 100644 --- a/libsanitizer/hwasan/hwasan_checks.h +++ b/libsanitizer/hwasan/hwasan_checks.h @@ -36,6 +36,15 @@ __attribute__((always_inline)) static void SigTrap(uptr p) { "int3\n" "nopl %c0(%%rax)\n" ::"n"(0x40 + X), "D"(p)); +#elif SANITIZER_RISCV64 + // Put pointer into x10 + // addiw contains immediate of 0x40 + X, where 0x40 is magic number and X + // encodes access size + register uptr x10 asm("x10") = p; + asm volatile( + "ebreak\n" + "addiw x0, x0, %1\n" ::"r"(x10), + "I"(0x40 + X)); #else // FIXME: not always sigill. __builtin_trap(); @@ -56,6 +65,14 @@ __attribute__((always_inline)) static void SigTrap(uptr p, uptr size) { "int3\n" "nopl %c0(%%rax)\n" ::"n"(0x40 + X), "D"(p), "S"(size)); +#elif SANITIZER_RISCV64 + // Put access size into x11 + register uptr x10 asm("x10") = p; + register uptr x11 asm("x11") = size; + asm volatile( + "ebreak\n" + "addiw x0, x0, %2\n" ::"r"(x10), + "r"(x11), "I"(0x40 + X)); #else __builtin_trap(); #endif @@ -71,7 +88,7 @@ __attribute__((always_inline, nodebug)) static bool PossiblyShortTagMatches( return false; if ((ptr & (kShadowAlignment - 1)) + sz > mem_tag) return false; -#ifndef __aarch64__ +#if !defined(__aarch64__) && !(SANITIZER_RISCV64) ptr = UntagAddr(ptr); #endif return *(u8 *)(ptr | (kShadowAlignment - 1)) == ptr_tag; diff --git a/libsanitizer/hwasan/hwasan_exceptions.cpp b/libsanitizer/hwasan/hwasan_exceptions.cpp index 6ed1da33542..c9968a5e360 100644 --- a/libsanitizer/hwasan/hwasan_exceptions.cpp +++ b/libsanitizer/hwasan/hwasan_exceptions.cpp @@ -56,6 +56,8 @@ __hwasan_personality_wrapper(int version, _Unwind_Action actions, uptr fp = get_gr(context, 6); // rbp #elif defined(__aarch64__) uptr fp = get_gr(context, 29); // x29 +#elif SANITIZER_RISCV64 + uptr fp = get_gr(context, 8); // x8 #else #error Unsupported architecture #endif diff --git a/libsanitizer/hwasan/hwasan_fuchsia.cpp b/libsanitizer/hwasan/hwasan_fuchsia.cpp index 94e5c5fb69c..967c796c339 100644 --- a/libsanitizer/hwasan/hwasan_fuchsia.cpp +++ b/libsanitizer/hwasan/hwasan_fuchsia.cpp @@ -15,6 +15,9 @@ #include "sanitizer_common/sanitizer_fuchsia.h" #if SANITIZER_FUCHSIA +#include <zircon/features.h> +#include <zircon/syscalls.h> + #include "hwasan.h" #include "hwasan_interface_internal.h" #include "hwasan_report.h" @@ -182,9 +185,20 @@ void InstallAtExitHandler() {} void HwasanInstallAtForkHandler() {} -// TODO(fxbug.dev/81499): Once we finalize the tagged pointer ABI in zircon, we should come back -// here and implement the appropriate check that TBI is enabled. -void InitializeOsSupport() {} +void InitializeOsSupport() { +#ifdef __aarch64__ + uint32_t features = 0; + CHECK_EQ(zx_system_get_features(ZX_FEATURE_KIND_ADDRESS_TAGGING, &features), + ZX_OK); + if (!(features & ZX_ARM64_FEATURE_ADDRESS_TAGGING_TBI) && + flags()->fail_without_syscall_abi) { + Printf( + "FATAL: HWAddressSanitizer requires " + "ZX_ARM64_FEATURE_ADDRESS_TAGGING_TBI.\n"); + Die(); + } +#endif +} } // namespace __hwasan diff --git a/libsanitizer/hwasan/hwasan_interceptors.cpp b/libsanitizer/hwasan/hwasan_interceptors.cpp index 8dc886e587e..c67927dc9f6 100644 --- a/libsanitizer/hwasan/hwasan_interceptors.cpp +++ b/libsanitizer/hwasan/hwasan_interceptors.cpp @@ -75,6 +75,8 @@ InternalLongjmp(__hw_register_buf env, int retval) { constexpr size_t kSpIndex = 13; # elif defined(__x86_64__) constexpr size_t kSpIndex = 6; +# elif SANITIZER_RISCV64 + constexpr size_t kSpIndex = 13; # endif // Clear all memory tags on the stack between here and where we're going. @@ -131,6 +133,49 @@ InternalLongjmp(__hw_register_buf env, int retval) { "cmovnz %1,%%rax;" "jmp *%%rdx;" ::"r"(env_address), "r"(retval_tmp)); +# elif SANITIZER_RISCV64 + register long int retval_tmp asm("x11") = retval; + register void *env_address asm("x10") = &env[0]; + asm volatile( + "ld ra, 0<<3(%0);" + "ld s0, 1<<3(%0);" + "ld s1, 2<<3(%0);" + "ld s2, 3<<3(%0);" + "ld s3, 4<<3(%0);" + "ld s4, 5<<3(%0);" + "ld s5, 6<<3(%0);" + "ld s6, 7<<3(%0);" + "ld s7, 8<<3(%0);" + "ld s8, 9<<3(%0);" + "ld s9, 10<<3(%0);" + "ld s10, 11<<3(%0);" + "ld s11, 12<<3(%0);" +# if __riscv_float_abi_double + "fld fs0, 14<<3(%0);" + "fld fs1, 15<<3(%0);" + "fld fs2, 16<<3(%0);" + "fld fs3, 17<<3(%0);" + "fld fs4, 18<<3(%0);" + "fld fs5, 19<<3(%0);" + "fld fs6, 20<<3(%0);" + "fld fs7, 21<<3(%0);" + "fld fs8, 22<<3(%0);" + "fld fs9, 23<<3(%0);" + "fld fs10, 24<<3(%0);" + "fld fs11, 25<<3(%0);" +# elif __riscv_float_abi_soft +# else +# error "Unsupported case" +# endif + "ld a4, 13<<3(%0);" + "mv sp, a4;" + // Return the value requested to return through arguments. + // This should be in x11 given what we requested above. + "seqz a0, %1;" + "add a0, a0, %1;" + "ret;" + : "+r"(env_address) + : "r"(retval_tmp)); # endif } diff --git a/libsanitizer/hwasan/hwasan_interface_internal.h b/libsanitizer/hwasan/hwasan_interface_internal.h index ef771add411..d1ecbb592a2 100644 --- a/libsanitizer/hwasan/hwasan_interface_internal.h +++ b/libsanitizer/hwasan/hwasan_interface_internal.h @@ -168,6 +168,14 @@ void __hwasan_thread_exit(); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_print_memory_usage(); +// The compiler will generate this when +// `-hwasan-record-stack-history-with-calls` is added as a flag, which will add +// frame record information to the stack ring buffer. This is an alternative to +// the compiler emitting instructions in the prologue for doing the same thing +// by accessing the ring buffer directly. +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_add_frame_record(u64 frame_record_info); + SANITIZER_INTERFACE_ATTRIBUTE void *__hwasan_memcpy(void *dst, const void *src, uptr size); SANITIZER_INTERFACE_ATTRIBUTE diff --git a/libsanitizer/hwasan/hwasan_linux.cpp b/libsanitizer/hwasan/hwasan_linux.cpp index ba9e23621cc..88ccfde007d 100644 --- a/libsanitizer/hwasan/hwasan_linux.cpp +++ b/libsanitizer/hwasan/hwasan_linux.cpp @@ -110,15 +110,84 @@ static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { FindDynamicShadowStart(shadow_size_bytes); } -void InitializeOsSupport() { +static void MaybeDieIfNoTaggingAbi(const char *message) { + if (!flags()->fail_without_syscall_abi) + return; + Printf("FATAL: %s\n", message); + Die(); +} + # define PR_SET_TAGGED_ADDR_CTRL 55 # define PR_GET_TAGGED_ADDR_CTRL 56 # define PR_TAGGED_ADDR_ENABLE (1UL << 0) +# define ARCH_GET_UNTAG_MASK 0x4001 +# define ARCH_ENABLE_TAGGED_ADDR 0x4002 +# define ARCH_GET_MAX_TAG_BITS 0x4003 + +static bool CanUseTaggingAbi() { +# if defined(__x86_64__) + unsigned long num_bits = 0; + // Check for x86 LAM support. This API is based on a currently unsubmitted + // patch to the Linux kernel (as of August 2022) and is thus subject to + // change. The patch is here: + // https://lore.kernel.org/all/20220815041803.17954-1-kirill.shutemov@linux.intel.com/ + // + // arch_prctl(ARCH_GET_MAX_TAG_BITS, &bits) returns the maximum number of tag + // bits the user can request, or zero if LAM is not supported by the hardware. + if (internal_iserror(internal_arch_prctl(ARCH_GET_MAX_TAG_BITS, + reinterpret_cast<uptr>(&num_bits)))) + return false; + // The platform must provide enough bits for HWASan tags. + if (num_bits < kTagBits) + return false; + return true; +# else + // Check for ARM TBI support. + return !internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)); +# endif // __x86_64__ +} + +static bool EnableTaggingAbi() { +# if defined(__x86_64__) + // Enable x86 LAM tagging for the process. + // + // arch_prctl(ARCH_ENABLE_TAGGED_ADDR, bits) enables tagging if the number of + // tag bits requested by the user does not exceed that provided by the system. + // arch_prctl(ARCH_GET_UNTAG_MASK, &mask) returns the mask of significant + // address bits. It is ~0ULL if either LAM is disabled for the process or LAM + // is not supported by the hardware. + if (internal_iserror(internal_arch_prctl(ARCH_ENABLE_TAGGED_ADDR, kTagBits))) + return false; + unsigned long mask = 0; + // Make sure the tag bits are where we expect them to be. + if (internal_iserror(internal_arch_prctl(ARCH_GET_UNTAG_MASK, + reinterpret_cast<uptr>(&mask)))) + return false; + // @mask has ones for non-tag bits, whereas @kAddressTagMask has ones for tag + // bits. Therefore these masks must not overlap. + if (mask & kAddressTagMask) + return false; + return true; +# else + // Enable ARM TBI tagging for the process. If for some reason tagging is not + // supported, prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE) returns + // -EINVAL. + if (internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, + PR_TAGGED_ADDR_ENABLE, 0, 0, 0))) + return false; + // Ensure that TBI is enabled. + if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) != + PR_TAGGED_ADDR_ENABLE) + return false; + return true; +# endif // __x86_64__ +} + +void InitializeOsSupport() { // Check we're running on a kernel that can use the tagged address ABI. - int local_errno = 0; - if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), - &local_errno) && - local_errno == EINVAL) { + bool has_abi = CanUseTaggingAbi(); + + if (!has_abi) { # if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE) // Some older Android kernels have the tagged pointer ABI on // unconditionally, and hence don't have the tagged-addr prctl while still @@ -127,46 +196,22 @@ void InitializeOsSupport() { // case. return; # else - if (flags()->fail_without_syscall_abi) { - Printf( - "FATAL: " - "HWAddressSanitizer requires a kernel with tagged address ABI.\n"); - Die(); - } + MaybeDieIfNoTaggingAbi( + "HWAddressSanitizer requires a kernel with tagged address ABI."); # endif } - // Turn on the tagged address ABI. - if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, - PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) || - !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) { -# if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) - // Try the new prctl API for Intel LAM. The API is based on a currently - // unsubmitted patch to the Linux kernel (as of May 2021) and is thus - // subject to change. Patch is here: - // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/ - int tag_bits = kTagBits; - int tag_shift = kAddressTagShift; - if (!internal_iserror( - internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, - reinterpret_cast<unsigned long>(&tag_bits), - reinterpret_cast<unsigned long>(&tag_shift), 0))) { - CHECK_EQ(tag_bits, kTagBits); - CHECK_EQ(tag_shift, kAddressTagShift); - return; - } -# endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) - if (flags()->fail_without_syscall_abi) { - Printf( - "FATAL: HWAddressSanitizer failed to enable tagged address syscall " - "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " - "configuration.\n"); - Die(); - } - } -# undef PR_SET_TAGGED_ADDR_CTRL -# undef PR_GET_TAGGED_ADDR_CTRL -# undef PR_TAGGED_ADDR_ENABLE + if (EnableTaggingAbi()) + return; + +# if SANITIZER_ANDROID + MaybeDieIfNoTaggingAbi( + "HWAddressSanitizer failed to enable tagged address syscall ABI.\n" + "Check the `sysctl abi.tagged_addr_disabled` configuration."); +# else + MaybeDieIfNoTaggingAbi( + "HWAddressSanitizer failed to enable tagged address syscall ABI.\n"); +# endif } bool InitShadow() { @@ -358,6 +403,47 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { const uptr size = size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log; +# elif SANITIZER_RISCV64 + // Access type is encoded in the instruction following EBREAK as + // ADDI x0, x0, [0x40 + 0xXY]. For Y == 0xF, access size is stored in + // X11 register. Access address is always in X10 register. + uptr pc = (uptr)uc->uc_mcontext.__gregs[REG_PC]; + uint8_t byte1 = *((u8 *)(pc + 0)); + uint8_t byte2 = *((u8 *)(pc + 1)); + uint8_t byte3 = *((u8 *)(pc + 2)); + uint8_t byte4 = *((u8 *)(pc + 3)); + uint32_t ebreak = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24)); + bool isFaultShort = false; + bool isEbreak = (ebreak == 0x100073); + bool isShortEbreak = false; +# if defined(__riscv_compressed) + isFaultShort = ((ebreak & 0x3) != 0x3); + isShortEbreak = ((ebreak & 0xffff) == 0x9002); +# endif + // faulted insn is not ebreak, not our case + if (!(isEbreak || isShortEbreak)) + return AccessInfo{}; + // advance pc to point after ebreak and reconstruct addi instruction + pc += isFaultShort ? 2 : 4; + byte1 = *((u8 *)(pc + 0)); + byte2 = *((u8 *)(pc + 1)); + byte3 = *((u8 *)(pc + 2)); + byte4 = *((u8 *)(pc + 3)); + // reconstruct instruction + uint32_t instr = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24)); + // check if this is really 32 bit instruction + // code is encoded in top 12 bits, since instruction is supposed to be with + // imm + const unsigned code = (instr >> 20) & 0xffff; + const uptr addr = uc->uc_mcontext.__gregs[10]; + const bool is_store = code & 0x10; + const bool recover = code & 0x20; + const unsigned size_log = code & 0xf; + if (size_log > 4 && size_log != 0xf) + return AccessInfo{}; // Not our case + const uptr size = + size_log == 0xf ? uc->uc_mcontext.__gregs[11] : 1U << size_log; + # else # error Unsupported architecture # endif @@ -376,6 +462,19 @@ static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { # if defined(__aarch64__) uc->uc_mcontext.pc += 4; # elif defined(__x86_64__) +# elif SANITIZER_RISCV64 + // pc points to EBREAK which is 2 bytes long + uint8_t *exception_source = (uint8_t *)(uc->uc_mcontext.__gregs[REG_PC]); + uint8_t byte1 = (uint8_t)(*(exception_source + 0)); + uint8_t byte2 = (uint8_t)(*(exception_source + 1)); + uint8_t byte3 = (uint8_t)(*(exception_source + 2)); + uint8_t byte4 = (uint8_t)(*(exception_source + 3)); + uint32_t faulted = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24)); + bool isFaultShort = false; +# if defined(__riscv_compressed) + isFaultShort = ((faulted & 0x3) != 0x3); +# endif + uc->uc_mcontext.__gregs[REG_PC] += isFaultShort ? 2 : 4; # else # error Unsupported architecture # endif diff --git a/libsanitizer/hwasan/hwasan_report.cpp b/libsanitizer/hwasan/hwasan_report.cpp index 66d3d155d40..fe769589186 100644 --- a/libsanitizer/hwasan/hwasan_report.cpp +++ b/libsanitizer/hwasan/hwasan_report.cpp @@ -746,7 +746,7 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, } // See the frame breakdown defined in __hwasan_tag_mismatch (from -// hwasan_tag_mismatch_aarch64.S). +// hwasan_tag_mismatch_{aarch64,riscv64}.S). void ReportRegisters(uptr *frame, uptr pc) { Printf("Registers where the failure occurred (pc %p):\n", pc); @@ -754,8 +754,13 @@ void ReportRegisters(uptr *frame, uptr pc) { // reduce the amount of logcat error messages printed. Each Printf() will // result in a new logcat line, irrespective of whether a newline is present, // and so we wish to reduce the number of Printf() calls we have to make. +#if defined(__aarch64__) Printf(" x0 %016llx x1 %016llx x2 %016llx x3 %016llx\n", frame[0], frame[1], frame[2], frame[3]); +#elif SANITIZER_RISCV64 + Printf(" sp %016llx x1 %016llx x2 %016llx x3 %016llx\n", + reinterpret_cast<u8 *>(frame) + 256, frame[1], frame[2], frame[3]); +#endif Printf(" x4 %016llx x5 %016llx x6 %016llx x7 %016llx\n", frame[4], frame[5], frame[6], frame[7]); Printf(" x8 %016llx x9 %016llx x10 %016llx x11 %016llx\n", @@ -770,8 +775,14 @@ void ReportRegisters(uptr *frame, uptr pc) { frame[24], frame[25], frame[26], frame[27]); // hwasan_check* reduces the stack pointer by 256, then __hwasan_tag_mismatch // passes it to this function. +#if defined(__aarch64__) Printf(" x28 %016llx x29 %016llx x30 %016llx sp %016llx\n", frame[28], frame[29], frame[30], reinterpret_cast<u8 *>(frame) + 256); +#elif SANITIZER_RISCV64 + Printf(" x28 %016llx x29 %016llx x30 %016llx x31 %016llx\n", frame[28], + frame[29], frame[30], frame[31]); +#else +#endif } } // namespace __hwasan diff --git a/libsanitizer/hwasan/hwasan_setjmp_riscv64.S b/libsanitizer/hwasan/hwasan_setjmp_riscv64.S new file mode 100644 index 00000000000..f33c4916df1 --- /dev/null +++ b/libsanitizer/hwasan/hwasan_setjmp_riscv64.S @@ -0,0 +1,97 @@ +//===-- hwasan_setjmp_riscv64.S -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// setjmp interceptor for risc-v. +// HWAddressSanitizer runtime. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_asm.h" +#include "builtins/assembly.h" + +#if HWASAN_WITH_INTERCEPTORS && defined(__riscv) && (__riscv_xlen == 64) +#include "sanitizer_common/sanitizer_platform.h" + +// We want to save the context of the calling function. +// That requires +// 1) No modification of the link register by this function. +// 2) No modification of the stack pointer by this function. +// 3) (no modification of any other saved register, but that's not really going +// to occur, and hence isn't as much of a worry). +// +// There's essentially no way to ensure that the compiler will not modify the +// stack pointer when compiling a C function. +// Hence we have to write this function in assembly. + +.section .text +.file "hwasan_setjmp_riscv64.S" + +.global __interceptor_setjmp +ASM_TYPE_FUNCTION(__interceptor_setjmp) +__interceptor_setjmp: + CFI_STARTPROC + addi x11, x0, 0 + j __interceptor_sigsetjmp + CFI_ENDPROC +ASM_SIZE(__interceptor_setjmp) + +.global __interceptor_sigsetjmp +ASM_TYPE_FUNCTION(__interceptor_sigsetjmp) +__interceptor_sigsetjmp: + CFI_STARTPROC + sd ra, 0<<3(x10) + sd s0, 1<<3(x10) + sd s1, 2<<3(x10) + sd s2, 3<<3(x10) + sd s3, 4<<3(x10) + sd s4, 5<<3(x10) + sd s5, 6<<3(x10) + sd s6, 7<<3(x10) + sd s7, 8<<3(x10) + sd s8, 9<<3(x10) + sd s9, 10<<3(x10) + sd s10, 11<<3(x10) + sd s11, 12<<3(x10) + sd sp, 13<<3(x10) +#if __riscv_float_abi_double + fsd fs0, 14<<3(x10) + fsd fs1, 15<<3(x10) + fsd fs2, 16<<3(x10) + fsd fs3, 17<<3(x10) + fsd fs4, 18<<3(x10) + fsd fs5, 19<<3(x10) + fsd fs6, 20<<3(x10) + fsd fs7, 21<<3(x10) + fsd fs8, 22<<3(x10) + fsd fs9, 23<<3(x10) + fsd fs10, 24<<3(x10) + fsd fs11, 25<<3(x10) +#elif __riscv_float_abi_soft +#else +# error "Unsupported case" +#endif + // We always have the second argument to __sigjmp_save (savemask) set, since + // the _setjmp function above has set it for us as `false`. + // This function is defined in hwasan_interceptors.cc + tail __sigjmp_save + CFI_ENDPROC +ASM_SIZE(__interceptor_sigsetjmp) + + +.macro WEAK_ALIAS first second + .weak \second + .equ \second\(), \first +.endm + +WEAK_ALIAS __interceptor_sigsetjmp, __sigsetjmp + +WEAK_ALIAS __interceptor_setjmp, _setjmp +#endif + +// We do not need executable stack. +NO_EXEC_STACK_DIRECTIVE diff --git a/libsanitizer/hwasan/hwasan_tag_mismatch_riscv64.S b/libsanitizer/hwasan/hwasan_tag_mismatch_riscv64.S new file mode 100644 index 00000000000..487a042405b --- /dev/null +++ b/libsanitizer/hwasan/hwasan_tag_mismatch_riscv64.S @@ -0,0 +1,132 @@ +#include "sanitizer_common/sanitizer_asm.h" + +// The content of this file is RISCV64-only: +#if defined(__riscv) && (__riscv_xlen == 64) + +// The responsibility of the HWASan entry point in compiler-rt is to primarily +// readjust the stack from the callee and save the current register values to +// the stack. +// This entry point function should be called from a __hwasan_check_* symbol. +// These are generated during a lowering pass in the backend, and are found in +// RISCVAsmPrinter::EmitHwasanMemaccessSymbols(). Please look there for +// further information. +// The __hwasan_check_* caller of this function should have expanded the stack +// and saved the previous values of x10(arg0), x11(arg1), x1(ra), and x8(fp). +// This function will "consume" these saved values and treats it as part of its +// own stack frame. In this sense, the __hwasan_check_* callee and this function +// "share" a stack frame. This allows us to omit having unwinding information +// (.cfi_*) present in every __hwasan_check_* function, therefore reducing binary size. +// This is particularly important as hwasan_check_* instances are duplicated in every +// translation unit where HWASan is enabled. +// This function calls HwasanTagMismatch to step back into the C++ code that +// completes the stack unwinding and error printing. This function is is not +// permitted to return. + + +// | ... | +// | ... | +// | Previous stack frames... | +// +=================================+ +// | ... | +// | | +// | Stack frame space for x12 - x31.| +// | | +// | ... | +// +---------------------------------+ <-- [SP + 96] +// | Saved x11(arg1), as | +// | __hwasan_check_* clobbers it. | +// +---------------------------------+ <-- [SP + 88] +// | Saved x10(arg0), as | +// | __hwasan_check_* clobbers it. | +// +---------------------------------+ <-- [SP + 80] +// | | +// | Stack frame space for x9. | +// +---------------------------------+ <-- [SP + 72] +// | | +// | Saved x8(fp), as | +// | __hwasan_check_* clobbers it. | +// +---------------------------------+ <-- [SP + 64] +// | ... | +// | | +// | Stack frame space for x2 - x7. | +// | | +// | ... | +// +---------------------------------+ <-- [SP + 16] +// | Return address (x1) for caller | +// | of __hwasan_check_*. | +// +---------------------------------+ <-- [SP + 8] +// | Reserved place for x0, possibly | +// | junk, since we don't save it. | +// +---------------------------------+ <-- [x2 / SP] + +// This function takes two arguments: +// * x10/a0: The data address. +// * x11/a1: The encoded access info for the failing access. + +.section .text +.file "hwasan_tag_mismatch_riscv64.S" + +.global __hwasan_tag_mismatch_v2 +ASM_TYPE_FUNCTION(__hwasan_tag_mismatch_v2) +__hwasan_tag_mismatch_v2: + CFI_STARTPROC + + // Set the CFA to be the return address for caller of __hwasan_check_*. Note + // that we do not emit CFI predicates to describe the contents of this stack + // frame, as this proxy entry point should never be debugged. The contents + // are static and are handled by the unwinder after calling + // __hwasan_tag_mismatch. The frame pointer is already correctly setup + // by __hwasan_check_*. + addi fp, sp, 256 + CFI_DEF_CFA(fp, 0) + CFI_OFFSET(ra, -248) + CFI_OFFSET(fp, -192) + + // Save the rest of the registers into the preallocated space left by + // __hwasan_check. + sd x31, 248(sp) + sd x30, 240(sp) + sd x29, 232(sp) + sd x28, 224(sp) + sd x27, 216(sp) + sd x26, 208(sp) + sd x25, 200(sp) + sd x24, 192(sp) + sd x23, 184(sp) + sd x22, 176(sp) + sd x21, 168(sp) + sd x20, 160(sp) + sd x19, 152(sp) + sd x18, 144(sp) + sd x17, 136(sp) + sd x16, 128(sp) + sd x15, 120(sp) + sd x14, 112(sp) + sd x13, 104(sp) + sd x12, 96(sp) + // sd x11, 88(sp) ; already saved + // sd x10, 80(sp) ; already saved + sd x9, 72(sp) + // sd x8, 64(sp) ; already saved + sd x7, 56(sp) + sd x6, 48(sp) + sd x5, 40(sp) + sd x4, 32(sp) + sd x3, 24(sp) + sd x2, 16(sp) + // sd x1, 8(sp) ; already saved + // sd x0, 0(sp) ; don't store zero register + + // Pass the address of the frame to __hwasan_tag_mismatch4, so that it can + // extract the saved registers from this frame without having to worry about + // finding this frame. + mv x12, sp + + call __hwasan_tag_mismatch4 + CFI_ENDPROC +ASM_SIZE(__hwasan_tag_mismatch_v2) + +#endif // defined(__riscv) && (__riscv_xlen == 64) + +// We do not need executable stack. +NO_EXEC_STACK_DIRECTIVE diff --git a/libsanitizer/interception/interception.h b/libsanitizer/interception/interception.h index d8dc092c45f..d97974ee907 100644 --- a/libsanitizer/interception/interception.h +++ b/libsanitizer/interception/interception.h @@ -16,7 +16,7 @@ #include "sanitizer_common/sanitizer_internal_defs.h" -#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \ +#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE && \ !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \ !SANITIZER_SOLARIS # error "Interception doesn't work on this operating system." @@ -88,7 +88,7 @@ typedef __sanitizer::OFF64_T OFF64_T; // As it's decided at compile time which functions are to be intercepted on Mac, // INTERCEPT_FUNCTION() is effectively a no-op on this system. -#if SANITIZER_MAC +#if SANITIZER_APPLE #include <sys/cdefs.h> // For __DARWIN_ALIAS_C(). // Just a pair of pointers. @@ -157,7 +157,7 @@ const interpose_substitution substitution_##func_name[] \ # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) # define REAL(x) __unsanitized_##x # define DECLARE_REAL(ret_type, func, ...) -#elif !SANITIZER_MAC +#elif !SANITIZER_APPLE # define PTR_TO_REAL(x) real_##x # define REAL(x) __interception::PTR_TO_REAL(x) # define FUNC_TYPE(x) x##_type @@ -168,12 +168,12 @@ const interpose_substitution substitution_##func_name[] \ extern FUNC_TYPE(func) PTR_TO_REAL(func); \ } # define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src) -#else // SANITIZER_MAC +#else // SANITIZER_APPLE # define REAL(x) x # define DECLARE_REAL(ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__); # define ASSIGN_REAL(x, y) -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE #if !SANITIZER_FUCHSIA # define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ @@ -193,7 +193,7 @@ const interpose_substitution substitution_##func_name[] \ // macros does its job. In exceptional cases you may need to call REAL(foo) // without defining INTERCEPTOR(..., foo, ...). For example, if you override // foo with an interceptor for other function. -#if !SANITIZER_MAC && !SANITIZER_FUCHSIA +#if !SANITIZER_APPLE && !SANITIZER_FUCHSIA # define DEFINE_REAL(ret_type, func, ...) \ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ namespace __interception { \ @@ -213,7 +213,7 @@ const interpose_substitution substitution_##func_name[] \ __interceptor_##func(__VA_ARGS__); \ extern "C" INTERCEPTOR_ATTRIBUTE ret_type func(__VA_ARGS__) -#elif !SANITIZER_MAC +#elif !SANITIZER_APPLE #define INTERCEPTOR(ret_type, func, ...) \ DEFINE_REAL(ret_type, func, __VA_ARGS__) \ @@ -226,7 +226,7 @@ const interpose_substitution substitution_##func_name[] \ #define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \ INTERCEPTOR(ret_type, func, __VA_ARGS__) -#else // SANITIZER_MAC +#else // SANITIZER_APPLE #define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__) suffix; \ @@ -278,7 +278,7 @@ typedef unsigned long uptr; # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) # define INTERCEPT_FUNCTION_VER(func, symver) \ INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) -#elif SANITIZER_MAC +#elif SANITIZER_APPLE # include "interception_mac.h" # define INTERCEPT_FUNCTION(func) INTERCEPT_FUNCTION_MAC(func) # define INTERCEPT_FUNCTION_VER(func, symver) \ diff --git a/libsanitizer/interception/interception_mac.cpp b/libsanitizer/interception/interception_mac.cpp index fb6eadcff59..03eae0fdca0 100644 --- a/libsanitizer/interception/interception_mac.cpp +++ b/libsanitizer/interception/interception_mac.cpp @@ -13,6 +13,6 @@ #include "interception.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/interception/interception_mac.h b/libsanitizer/interception/interception_mac.h index eddedb8959c..26079518c64 100644 --- a/libsanitizer/interception/interception_mac.h +++ b/libsanitizer/interception/interception_mac.h @@ -11,7 +11,7 @@ // Mac-specific interception methods. //===----------------------------------------------------------------------===// -#if SANITIZER_MAC +#if SANITIZER_APPLE #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) # error "interception_mac.h should be included from interception.h only" @@ -24,4 +24,4 @@ #define INTERCEPT_FUNCTION_VER_MAC(func, symver) #endif // INTERCEPTION_MAC_H -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/interception/interception_type_test.cpp b/libsanitizer/interception/interception_type_test.cpp index a611604a700..2a118fb214f 100644 --- a/libsanitizer/interception/interception_type_test.cpp +++ b/libsanitizer/interception/interception_type_test.cpp @@ -13,7 +13,7 @@ #include "interception.h" -#if SANITIZER_LINUX || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_APPLE #include <sys/types.h> #include <stddef.h> @@ -24,7 +24,7 @@ COMPILER_CHECK(sizeof(::SSIZE_T) == sizeof(ssize_t)); COMPILER_CHECK(sizeof(::PTRDIFF_T) == sizeof(ptrdiff_t)); COMPILER_CHECK(sizeof(::INTMAX_T) == sizeof(intmax_t)); -#if !SANITIZER_MAC +#if !SANITIZER_APPLE COMPILER_CHECK(sizeof(::OFF64_T) == sizeof(off64_t)); #endif diff --git a/libsanitizer/interception/interception_win.cpp b/libsanitizer/interception/interception_win.cpp index 10b893391f4..d0db981d519 100644 --- a/libsanitizer/interception/interception_win.cpp +++ b/libsanitizer/interception/interception_win.cpp @@ -1068,4 +1068,4 @@ bool OverrideImportedFunction(const char *module_to_patch, } // namespace __interception -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/lsan/lsan_allocator.cpp b/libsanitizer/lsan/lsan_allocator.cpp index b4fd7e904be..43928ad294e 100644 --- a/libsanitizer/lsan/lsan_allocator.cpp +++ b/libsanitizer/lsan/lsan_allocator.cpp @@ -146,6 +146,8 @@ void GetAllocatorCacheRange(uptr *begin, uptr *end) { } uptr GetMallocUsableSize(const void *p) { + if (!p) + return 0; ChunkMetadata *m = Metadata(p); if (!m) return 0; return m->requested_size; diff --git a/libsanitizer/lsan/lsan_allocator.h b/libsanitizer/lsan/lsan_allocator.h index 539330491b0..b67d9d7750e 100644 --- a/libsanitizer/lsan/lsan_allocator.h +++ b/libsanitizer/lsan/lsan_allocator.h @@ -49,8 +49,7 @@ struct ChunkMetadata { u32 stack_trace_id; }; -#if defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \ - defined(__arm__) || SANITIZER_RISCV64 || defined(__hexagon__) +#if !SANITIZER_CAN_USE_ALLOCATOR64 template <typename AddressSpaceViewTy> struct AP32 { static const uptr kSpaceBeg = 0; @@ -65,7 +64,7 @@ struct AP32 { template <typename AddressSpaceView> using PrimaryAllocatorASVT = SizeClassAllocator32<AP32<AddressSpaceView>>; using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; -#elif defined(__x86_64__) || defined(__powerpc64__) || defined(__s390x__) +#else # if SANITIZER_FUCHSIA || defined(__powerpc64__) const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. diff --git a/libsanitizer/lsan/lsan_common.cpp b/libsanitizer/lsan/lsan_common.cpp index 8d1bf11fdab..94bb3cca008 100644 --- a/libsanitizer/lsan/lsan_common.cpp +++ b/libsanitizer/lsan/lsan_common.cpp @@ -105,7 +105,7 @@ static const char kStdSuppressions[] = // definition. "leak:*pthread_exit*\n" # endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT -# if SANITIZER_MAC +# if SANITIZER_APPLE // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173 "leak:*_os_trace*\n" # endif @@ -240,7 +240,7 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator { const char *Leak() { return Blue(); } }; -static inline bool CanBeAHeapPointer(uptr p) { +static inline bool MaybeUserPointer(uptr p) { // Since our heap is located in mmap-ed memory, we can assume a sensible lower // bound on heap addresses. const uptr kMinAddress = 4 * 4096; @@ -252,8 +252,8 @@ static inline bool CanBeAHeapPointer(uptr p) { # elif defined(__mips64) return ((p >> 40) == 0); # elif defined(__aarch64__) - unsigned runtimeVMA = (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); - return ((p >> runtimeVMA) == 0); + // Accept up to 48 bit VMA. + return ((p >> 48) == 0); # else return true; # endif @@ -276,7 +276,7 @@ void ScanRangeForPointers(uptr begin, uptr end, Frontier *frontier, pp = pp + alignment - pp % alignment; for (; pp + sizeof(void *) <= end; pp += alignment) { void *p = *reinterpret_cast<void **>(pp); - if (!CanBeAHeapPointer(reinterpret_cast<uptr>(p))) + if (!MaybeUserPointer(reinterpret_cast<uptr>(p))) continue; uptr chunk = PointsIntoChunk(p); if (!chunk) @@ -949,7 +949,7 @@ void __lsan_ignore_object(const void *p) { Lock l(&global_mutex); IgnoreObjectResult res = IgnoreObjectLocked(p); if (res == kIgnoreObjectInvalid) - VReport(1, "__lsan_ignore_object(): no heap object found at %p", p); + VReport(1, "__lsan_ignore_object(): no heap object found at %p\n", p); if (res == kIgnoreObjectAlreadyIgnored) VReport(1, "__lsan_ignore_object(): " @@ -1032,13 +1032,11 @@ SANITIZER_INTERFACE_WEAK_DEF(const char *, __lsan_default_options, void) { } #if !SANITIZER_SUPPORTS_WEAK_HOOKS -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int -__lsan_is_turned_off() { +SANITIZER_INTERFACE_WEAK_DEF(int, __lsan_is_turned_off, void) { return 0; } -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char * -__lsan_default_suppressions() { +SANITIZER_INTERFACE_WEAK_DEF(const char *, __lsan_default_suppressions, void) { return ""; } #endif diff --git a/libsanitizer/lsan/lsan_common.h b/libsanitizer/lsan/lsan_common.h index 6b06c4517cd..d7153751fae 100644 --- a/libsanitizer/lsan/lsan_common.h +++ b/libsanitizer/lsan/lsan_common.h @@ -34,11 +34,11 @@ // is missing. This caused a link error. #if SANITIZER_ANDROID && (__ANDROID_API__ < 28 || defined(__arm__)) # define CAN_SANITIZE_LEAKS 0 -#elif (SANITIZER_LINUX || SANITIZER_MAC) && (SANITIZER_WORDSIZE == 64) && \ +#elif (SANITIZER_LINUX || SANITIZER_APPLE) && (SANITIZER_WORDSIZE == 64) && \ (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__) || \ defined(__powerpc64__) || defined(__s390x__)) # define CAN_SANITIZE_LEAKS 1 -#elif defined(__i386__) && (SANITIZER_LINUX || SANITIZER_MAC) +#elif defined(__i386__) && (SANITIZER_LINUX || SANITIZER_APPLE) # define CAN_SANITIZE_LEAKS 1 #elif defined(__arm__) && SANITIZER_LINUX # define CAN_SANITIZE_LEAKS 1 diff --git a/libsanitizer/lsan/lsan_common_mac.cpp b/libsanitizer/lsan/lsan_common_mac.cpp index a4204740c7f..26b623fb1d4 100644 --- a/libsanitizer/lsan/lsan_common_mac.cpp +++ b/libsanitizer/lsan/lsan_common_mac.cpp @@ -15,7 +15,7 @@ #include "sanitizer_common/sanitizer_libc.h" #include "lsan_common.h" -#if CAN_SANITIZE_LEAKS && SANITIZER_MAC +#if CAN_SANITIZE_LEAKS && SANITIZER_APPLE #include "sanitizer_common/sanitizer_allocator_internal.h" #include "lsan_allocator.h" @@ -201,4 +201,4 @@ void LockStuffAndStopTheWorld(StopTheWorldCallback callback, } // namespace __lsan -#endif // CAN_SANITIZE_LEAKS && SANITIZER_MAC +#endif // CAN_SANITIZE_LEAKS && SANITIZER_APPLE diff --git a/libsanitizer/lsan/lsan_interceptors.cpp b/libsanitizer/lsan/lsan_interceptors.cpp index 205e85685a7..3a1b2afdbb7 100644 --- a/libsanitizer/lsan/lsan_interceptors.cpp +++ b/libsanitizer/lsan/lsan_interceptors.cpp @@ -67,7 +67,7 @@ namespace std { enum class align_val_t: size_t; } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE INTERCEPTOR(void*, malloc, uptr size) { if (DlsymAlloc::Use()) return DlsymAlloc::Allocate(size); @@ -116,7 +116,7 @@ INTERCEPTOR(void*, valloc, uptr size) { GET_STACK_TRACE_MALLOC; return lsan_valloc(size, stack); } -#endif // !SANITIZER_MAC +#endif // !SANITIZER_APPLE #if SANITIZER_INTERCEPT_MEMALIGN INTERCEPTOR(void*, memalign, uptr alignment, uptr size) { @@ -242,7 +242,7 @@ INTERCEPTOR(int, mprobe, void *ptr) { // libstdc++, each of has its implementation of new and delete. // To make sure that C++ allocation/deallocation operators are overridden on // OS X we need to intercept them using their mangled names. -#if !SANITIZER_MAC +#if !SANITIZER_APPLE INTERCEPTOR_ATTRIBUTE void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } @@ -301,7 +301,7 @@ INTERCEPTOR_ATTRIBUTE void operator delete[](void *ptr, size_t size, std::align_val_t) NOEXCEPT { OPERATOR_DELETE_BODY; } -#else // SANITIZER_MAC +#else // SANITIZER_APPLE INTERCEPTOR(void *, _Znwm, size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); } @@ -321,7 +321,7 @@ INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } -#endif // !SANITIZER_MAC +#endif // !SANITIZER_APPLE ///// Thread initialization and finalization. ///// diff --git a/libsanitizer/lsan/lsan_mac.cpp b/libsanitizer/lsan/lsan_mac.cpp index 10a73f8fa93..6964a9ba28d 100644 --- a/libsanitizer/lsan/lsan_mac.cpp +++ b/libsanitizer/lsan/lsan_mac.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "interception/interception.h" #include "lsan.h" @@ -188,4 +188,4 @@ INTERCEPTOR(void, dispatch_source_set_event_handler, dispatch_source_t ds, } #endif -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/lsan/lsan_malloc_mac.cpp b/libsanitizer/lsan/lsan_malloc_mac.cpp index d03eb2e915c..525c30272cc 100644 --- a/libsanitizer/lsan/lsan_malloc_mac.cpp +++ b/libsanitizer/lsan/lsan_malloc_mac.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "lsan.h" #include "lsan_allocator.h" @@ -56,4 +56,4 @@ using namespace __lsan; #include "sanitizer_common/sanitizer_malloc_mac.inc" -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_report.cpp b/libsanitizer/sanitizer_common/sanitizer_allocator_report.cpp index 1c6520819ef..129f925e6fb 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_report.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_report.cpp @@ -128,8 +128,7 @@ void NORETURN ReportAllocationSizeTooBig(uptr user_size, uptr max_size, void NORETURN ReportOutOfMemory(uptr requested_size, const StackTrace *stack) { { ScopedAllocatorErrorReport report("out-of-memory", stack); - Report("ERROR: %s: allocator is out of memory trying to allocate 0x%zx " - "bytes\n", SanitizerToolName, requested_size); + ERROR_OOM("allocator is trying to allocate 0x%zx bytes\n", requested_size); } Die(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.cpp b/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.cpp index 472b83d63a0..e0e2bd01069 100644 --- a/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.cpp @@ -143,4 +143,6 @@ void ChainedOriginDepot::LockAll() { depot.LockAll(); } void ChainedOriginDepot::UnlockAll() { depot.UnlockAll(); } +void ChainedOriginDepot::TestOnlyUnmap() { depot.TestOnlyUnmap(); } + } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.h b/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.h index 2e800964a45..f9f192b6857 100644 --- a/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.h +++ b/libsanitizer/sanitizer_common/sanitizer_chained_origin_depot.h @@ -34,6 +34,7 @@ class ChainedOriginDepot { void LockAll(); void UnlockAll(); + void TestOnlyUnmap(); private: ChainedOriginDepot(const ChainedOriginDepot &) = delete; diff --git a/libsanitizer/sanitizer_common/sanitizer_common.cpp b/libsanitizer/sanitizer_common/sanitizer_common.cpp index e30a93da5b5..82236453157 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_common.cpp @@ -46,9 +46,15 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, Die(); } recursion_count++; - Report("ERROR: %s failed to " - "%s 0x%zx (%zd) bytes of %s (error code: %d)\n", - SanitizerToolName, mmap_type, size, size, mem_type, err); + if (ErrorIsOOM(err)) { + ERROR_OOM("failed to %s 0x%zx (%zd) bytes of %s (error code: %d)\n", + mmap_type, size, size, mem_type, err); + } else { + Report( + "ERROR: %s failed to " + "%s 0x%zx (%zd) bytes of %s (error code: %d)\n", + SanitizerToolName, mmap_type, size, size, mem_type, err); + } #if !SANITIZER_GO DumpProcessMap(); #endif @@ -351,6 +357,13 @@ void SleepForSeconds(unsigned seconds) { } void SleepForMillis(unsigned millis) { internal_usleep((u64)millis * 1000); } +void WaitForDebugger(unsigned seconds, const char *label) { + if (seconds) { + Report("Sleeping for %u second(s) %s\n", seconds, label); + SleepForSeconds(seconds); + } +} + } // namespace __sanitizer using namespace __sanitizer; diff --git a/libsanitizer/sanitizer_common/sanitizer_common.h b/libsanitizer/sanitizer_common/sanitizer_common.h index 17570d60688..08c6062ba06 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common.h +++ b/libsanitizer/sanitizer_common/sanitizer_common.h @@ -120,6 +120,11 @@ bool MprotectReadOnly(uptr addr, uptr size); void MprotectMallocZones(void *addr, int prot); +#if SANITIZER_WINDOWS +// Zero previously mmap'd memory. Currently used only on Windows. +bool ZeroMmapFixedRegion(uptr fixed_addr, uptr size) WARN_UNUSED_RESULT; +#endif + #if SANITIZER_LINUX // Unmap memory. Currently only used on Linux. void UnmapFromTo(uptr from, uptr to); @@ -294,6 +299,7 @@ void InitTlsSize(); uptr GetTlsSize(); // Other +void WaitForDebugger(unsigned seconds, const char *label); void SleepForSeconds(unsigned seconds); void SleepForMillis(unsigned millis); u64 NanoTime(); @@ -310,6 +316,18 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, const char *mmap_type, error_t err, bool raw_report = false); +// Returns true if the platform-specific error reported is an OOM error. +bool ErrorIsOOM(error_t err); + +// This reports an error in the form: +// +// `ERROR: {{SanitizerToolName}}: out of memory: {{err_msg}}` +// +// Downstream tools that read sanitizer output will know that errors starting +// in this format are specifically OOM errors. +#define ERROR_OOM(err_msg, ...) \ + Report("ERROR: %s: out of memory: " err_msg, SanitizerToolName, __VA_ARGS__) + // Specific tools may override behavior of "Die" function to do tool-specific // job. typedef void (*DieCallbackType)(void); @@ -890,13 +908,13 @@ void WriteToSyslog(const char *buffer); #define SANITIZER_WIN_TRACE 0 #endif -#if SANITIZER_MAC || SANITIZER_WIN_TRACE +#if SANITIZER_APPLE || SANITIZER_WIN_TRACE void LogFullErrorReport(const char *buffer); #else inline void LogFullErrorReport(const char *buffer) {} #endif -#if SANITIZER_LINUX || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_APPLE void WriteOneLineToSyslog(const char *s); void LogMessageOnPrintf(const char *str); #else @@ -1003,7 +1021,6 @@ struct SignalContext { }; void InitializePlatformEarly(); -void MaybeReexec(); template <typename Fn> class RunOnDestruction { diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc index 43296e6c1f6..cd9235e503b 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc @@ -203,13 +203,13 @@ extern const short *_tolower_tab_; #endif // Platform-specific options. -#if SANITIZER_MAC +#if SANITIZER_APPLE #define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 #elif SANITIZER_WINDOWS64 #define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 #else #define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1 -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE #ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE #define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {} @@ -385,9 +385,11 @@ extern const short *_tolower_tab_; if (common_flags()->intercept_strndup) { \ COMMON_INTERCEPTOR_READ_STRING(ctx, s, Min(size, copy_length + 1)); \ } \ - COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \ - internal_memcpy(new_mem, s, copy_length); \ - new_mem[copy_length] = '\0'; \ + if (new_mem) { \ + COMMON_INTERCEPTOR_COPY_STRING(ctx, new_mem, s, copy_length); \ + internal_memcpy(new_mem, s, copy_length); \ + new_mem[copy_length] = '\0'; \ + } \ return new_mem; #endif @@ -1334,7 +1336,7 @@ INTERCEPTOR_WITH_SUFFIX(int, fputs, char *s, void *file) { // libc file streams can call user-supplied functions, see fopencookie. void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fputs, s, file); - if (!SANITIZER_MAC || s) { // `fputs(NULL, file)` is supported on Darwin. + if (!SANITIZER_APPLE || s) { // `fputs(NULL, file)` is supported on Darwin. COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1); } return REAL(fputs)(s, file); @@ -1349,7 +1351,7 @@ INTERCEPTOR(int, puts, char *s) { // libc file streams can call user-supplied functions, see fopencookie. void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, puts, s); - if (!SANITIZER_MAC || s) { // `puts(NULL)` is supported on Darwin. + if (!SANITIZER_APPLE || s) { // `puts(NULL)` is supported on Darwin. COMMON_INTERCEPTOR_READ_RANGE(ctx, s, internal_strlen(s) + 1); } return REAL(puts)(s); @@ -1365,8 +1367,13 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3, void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, prctl, option, arg2, arg3, arg4, arg5); static const int PR_SET_NAME = 15; + static const int PR_SET_VMA = 0x53564d41; static const int PR_SCHED_CORE = 62; static const int PR_SCHED_CORE_GET = 0; + if (option == PR_SET_VMA && arg2 == 0UL) { + char *name = (char *)arg5; + COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1); + } int res = REAL(prctl(option, arg2, arg3, arg4, arg5)); if (option == PR_SET_NAME) { char buff[16]; @@ -1952,7 +1959,7 @@ UNUSED static void unpoison_passwd(void *ctx, __sanitizer_passwd *pwd) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_gecos, internal_strlen(pwd->pw_gecos) + 1); #endif -#if SANITIZER_MAC || SANITIZER_FREEBSD || SANITIZER_NETBSD +#if SANITIZER_APPLE || SANITIZER_FREEBSD || SANITIZER_NETBSD if (pwd->pw_class) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, pwd->pw_class, internal_strlen(pwd->pw_class) + 1); @@ -2516,13 +2523,61 @@ INTERCEPTOR(int, __b64_pton, char const *src, char *target, SIZE_T targsize) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, target, res); return res; } -# define INIT___B64_TO \ +#define INIT___B64_TO \ COMMON_INTERCEPT_FUNCTION(__b64_ntop); \ COMMON_INTERCEPT_FUNCTION(__b64_pton); #else // SANITIZER_INTERCEPT___B64_TO #define INIT___B64_TO #endif // SANITIZER_INTERCEPT___B64_TO +#if SANITIZER_INTERCEPT_DN_COMP_EXPAND +# if __GLIBC_PREREQ(2, 34) +// Changed with https://sourceware.org/git/?p=glibc.git;h=640bbdf +# define DN_COMP_INTERCEPTOR_NAME dn_comp +# define DN_EXPAND_INTERCEPTOR_NAME dn_expand +# else +# define DN_COMP_INTERCEPTOR_NAME __dn_comp +# define DN_EXPAND_INTERCEPTOR_NAME __dn_expand +# endif +INTERCEPTOR(int, DN_COMP_INTERCEPTOR_NAME, unsigned char *exp_dn, + unsigned char *comp_dn, int length, unsigned char **dnptrs, + unsigned char **lastdnptr) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, DN_COMP_INTERCEPTOR_NAME, exp_dn, comp_dn, + length, dnptrs, lastdnptr); + int res = REAL(DN_COMP_INTERCEPTOR_NAME)(exp_dn, comp_dn, length, dnptrs, + lastdnptr); + if (res >= 0) { + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, comp_dn, res); + if (dnptrs && lastdnptr) { + unsigned char **p = dnptrs; + for (; p != lastdnptr && *p; ++p) + ; + if (p != lastdnptr) + ++p; + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dnptrs, (p - dnptrs) * sizeof(*p)); + } + } + return res; +} +INTERCEPTOR(int, DN_EXPAND_INTERCEPTOR_NAME, unsigned char const *base, + unsigned char const *end, unsigned char const *src, char *dest, + int space) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, DN_EXPAND_INTERCEPTOR_NAME, base, end, src, + dest, space); + // TODO: add read check if __dn_comp intercept added + int res = REAL(DN_EXPAND_INTERCEPTOR_NAME)(base, end, src, dest, space); + if (res >= 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, internal_strlen(dest) + 1); + return res; +} +# define INIT_DN_COMP_EXPAND \ + COMMON_INTERCEPT_FUNCTION(DN_COMP_INTERCEPTOR_NAME); \ + COMMON_INTERCEPT_FUNCTION(DN_EXPAND_INTERCEPTOR_NAME); +#else // SANITIZER_INTERCEPT_DN_COMP_EXPAND +# define INIT_DN_COMP_EXPAND +#endif // SANITIZER_INTERCEPT_DN_COMP_EXPAND #if SANITIZER_INTERCEPT_POSIX_SPAWN @@ -3941,7 +3996,7 @@ INTERCEPTOR(char *, strerror, int errnum) { // * GNU version returns message pointer, which points to either buf or some // static storage. #if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE) || \ - SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD || \ + SANITIZER_APPLE || SANITIZER_ANDROID || SANITIZER_NETBSD || \ SANITIZER_FREEBSD // POSIX version. Spec is not clear on whether buf is NULL-terminated. // At least on OSX, buf contents are valid even when the call fails. @@ -3974,7 +4029,7 @@ INTERCEPTOR(char *, strerror_r, int errnum, char *buf, SIZE_T buflen) { return res; } #endif //(_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE || - //SANITIZER_MAC + //SANITIZER_APPLE #define INIT_STRERROR_R COMMON_INTERCEPT_FUNCTION(strerror_r); #else #define INIT_STRERROR_R @@ -4943,6 +4998,27 @@ INTERCEPTOR(int, pthread_attr_getaffinity_np, void *attr, SIZE_T cpusetsize, #define INIT_PTHREAD_ATTR_GETAFFINITY_NP #endif +#if SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP +INTERCEPTOR(int, pthread_getaffinity_np, void *attr, SIZE_T cpusetsize, + void *cpuset) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pthread_getaffinity_np, attr, cpusetsize, + cpuset); + // FIXME: under ASan the call below may write to freed memory and corrupt + // its metadata. See + // https://github.com/google/sanitizers/issues/321. + int res = REAL(pthread_getaffinity_np)(attr, cpusetsize, cpuset); + if (!res && cpusetsize && cpuset) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cpuset, cpusetsize); + return res; +} + +#define INIT_PTHREAD_GETAFFINITY_NP \ + COMMON_INTERCEPT_FUNCTION(pthread_getaffinity_np); +#else +#define INIT_PTHREAD_GETAFFINITY_NP +#endif + #if SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED INTERCEPTOR_PTHREAD_MUTEXATTR_GET(pshared, sizeof(int)) #define INIT_PTHREAD_MUTEXATTR_GETPSHARED \ @@ -10308,6 +10384,42 @@ INTERCEPTOR(int, sigaltstack, void *ss, void *oss) { #define INIT_SIGALTSTACK #endif +#if SANITIZER_INTERCEPT_PROCCTL +INTERCEPTOR(int, procctl, int idtype, u64 id, int cmd, uptr data) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, procctl, idtype, id, cmd, data); + static const int PROC_REAP_ACQUIRE = 2; + static const int PROC_REAP_RELEASE = 3; + static const int PROC_REAP_STATUS = 4; + static const int PROC_REAP_GETPIDS = 5; + static const int PROC_REAP_KILL = 6; + if (cmd < PROC_REAP_ACQUIRE || cmd > PROC_REAP_KILL) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)data, sizeof(int)); + } else { + // reap_acquire/reap_release bears no arguments. + if (cmd > PROC_REAP_RELEASE) { + unsigned int reapsz; + switch (cmd) { + case PROC_REAP_STATUS: + reapsz = struct_procctl_reaper_status_sz; + break; + case PROC_REAP_GETPIDS: + reapsz = struct_procctl_reaper_pids_sz; + break; + case PROC_REAP_KILL: + reapsz = struct_procctl_reaper_kill_sz; + break; + } + COMMON_INTERCEPTOR_READ_RANGE(ctx, (void *)data, reapsz); + } + } + return REAL(procctl)(idtype, id, cmd, data); +} +#define INIT_PROCCTL COMMON_INTERCEPT_FUNCTION(procctl) +#else +#define INIT_PROCCTL +#endif + #if SANITIZER_INTERCEPT_UNAME INTERCEPTOR(int, uname, struct utsname *utsname) { #if SANITIZER_LINUX @@ -10425,6 +10537,7 @@ static void InitializeCommonInterceptors() { INIT_GLOB; INIT_GLOB64; INIT___B64_TO; + INIT_DN_COMP_EXPAND; INIT_POSIX_SPAWN; INIT_WAIT; INIT_WAIT4; @@ -10514,6 +10627,7 @@ static void InitializeCommonInterceptors() { INIT_PTHREAD_ATTR_GET_SCHED; INIT_PTHREAD_ATTR_GETINHERITSCHED; INIT_PTHREAD_ATTR_GETAFFINITY_NP; + INIT_PTHREAD_GETAFFINITY_NP; INIT_PTHREAD_MUTEXATTR_GETPSHARED; INIT_PTHREAD_MUTEXATTR_GETTYPE; INIT_PTHREAD_MUTEXATTR_GETPROTOCOL; @@ -10665,6 +10779,7 @@ static void InitializeCommonInterceptors() { INIT_QSORT_R; INIT_BSEARCH; INIT_SIGALTSTACK; + INIT_PROCCTL INIT_UNAME; INIT___XUNAME; diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S new file mode 100644 index 00000000000..05192485d59 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S @@ -0,0 +1,63 @@ +#if defined(__loongarch_lp64) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) +ASM_HIDDEN(_ZN14__interception10real_vforkE) + +.bss +.type _ZN14__interception10real_vforkE, @object +.size _ZN14__interception10real_vforkE, 8 +_ZN14__interception10real_vforkE: + .zero 8 + +.text +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save ra in the off-stack spill area. + // allocate space on stack + addi.d $sp, $sp, -16 + // store $ra value + st.d $ra, $sp, 8 + bl COMMON_INTERCEPTOR_SPILL_AREA + // restore previous values from stack + ld.d $ra, $sp, 8 + // adjust stack + addi.d $sp, $sp, 16 + // store $ra by $a0 + st.d $ra, $a0, 0 + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + la.local $a0, _ZN14__interception10real_vforkE + ld.d $a0, $a0, 0 + jirl $ra, $a0, 0 + + // adjust stack + addi.d $sp, $sp, -16 + // store $a0 by adjusted stack + st.d $a0, $sp, 8 + // jump to exit label if $a0 is 0 + beqz $a0, .L_exit + + // $a0 != 0 => parent process. Clear stack shadow. + // put old $sp to $a0 + addi.d $a0, $sp, 16 + bl %plt(COMMON_INTERCEPTOR_HANDLE_VFORK) + +.L_exit: + // Restore $ra + bl COMMON_INTERCEPTOR_SPILL_AREA + ld.d $ra, $a0, 0 + // load value by stack + ld.d $a0, $sp, 8 + // adjust stack + addi.d $sp, $sp, 16 + jr $ra +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp b/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp index a20602d8b95..67e77a87778 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp @@ -28,7 +28,7 @@ void Abort() { internal__exit(1); } bool CreateDir(const char *pathname) { return false; } #endif // !SANITIZER_WINDOWS -#if !SANITIZER_WINDOWS && !SANITIZER_MAC +#if !SANITIZER_WINDOWS && !SANITIZER_APPLE void ListOfModules::init() {} void InitializePlatformCommonFlags(CommonFlags *cf) {} #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_interface.inc b/libsanitizer/sanitizer_common/sanitizer_coverage_interface.inc index d7ab0c3d98c..9d36a40270d 100644 --- a/libsanitizer/sanitizer_common/sanitizer_coverage_interface.inc +++ b/libsanitizer/sanitizer_common/sanitizer_coverage_interface.inc @@ -27,6 +27,16 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load1) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load2) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load4) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load8) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load16) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store1) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store2) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store4) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store8) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store16) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init) diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep_new.cpp b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep_new.cpp index 3dcb39f32f6..956b48e0b43 100644 --- a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep_new.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep_new.cpp @@ -259,6 +259,16 @@ SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load1, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load2, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load4, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load8, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load16, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store1, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store2, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store4, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store8, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store16, void){} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init, char* start, char* end) { __sancov::SingletonCounterCoverage::Cov8bitCountersInit(start, end); diff --git a/libsanitizer/sanitizer_common/sanitizer_errno.h b/libsanitizer/sanitizer_common/sanitizer_errno.h index 70a6e88dbaa..46c85364cef 100644 --- a/libsanitizer/sanitizer_common/sanitizer_errno.h +++ b/libsanitizer/sanitizer_common/sanitizer_errno.h @@ -21,7 +21,7 @@ #include "sanitizer_errno_codes.h" #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_APPLE # define __errno_location __error #elif SANITIZER_ANDROID || SANITIZER_NETBSD # define __errno_location __errno diff --git a/libsanitizer/sanitizer_common/sanitizer_flags.inc b/libsanitizer/sanitizer_common/sanitizer_flags.inc index 0ca91aff8dd..6148ae56067 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flags.inc +++ b/libsanitizer/sanitizer_common/sanitizer_flags.inc @@ -62,16 +62,19 @@ COMMON_FLAG( COMMON_FLAG(const char *, log_suffix, nullptr, "String to append to log file name, e.g. \".txt\".") COMMON_FLAG( - bool, log_to_syslog, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC, + bool, log_to_syslog, (bool)SANITIZER_ANDROID || (bool)SANITIZER_APPLE, "Write all sanitizer output to syslog in addition to other means of " "logging.") COMMON_FLAG( int, verbosity, 0, "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).") -COMMON_FLAG(bool, strip_env, 1, +COMMON_FLAG(bool, strip_env, true, "Whether to remove the sanitizer from DYLD_INSERT_LIBRARIES to " - "avoid passing it to children. Default is true.") -COMMON_FLAG(bool, detect_leaks, !SANITIZER_MAC, "Enable memory leak detection.") + "avoid passing it to children on Apple platforms. Default is true.") +COMMON_FLAG(bool, verify_interceptors, true, + "Verify that interceptors are working on Apple platforms. Default " + "is true.") +COMMON_FLAG(bool, detect_leaks, !SANITIZER_APPLE, "Enable memory leak detection.") COMMON_FLAG( bool, leak_check_at_exit, true, "Invoke leak checking in an atexit handler. Has no effect if " @@ -245,7 +248,7 @@ COMMON_FLAG(bool, decorate_proc_maps, (bool)SANITIZER_ANDROID, COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool " "found an error") COMMON_FLAG( - bool, abort_on_error, (bool)SANITIZER_ANDROID || (bool)SANITIZER_MAC, + bool, abort_on_error, (bool)SANITIZER_ANDROID || (bool)SANITIZER_APPLE, "If set, the tool calls abort() instead of _exit() after printing the " "error report.") COMMON_FLAG(bool, suppress_equal_pcs, true, diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp index 848953a6ab0..a92e84cb8ec 100644 --- a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp @@ -32,7 +32,7 @@ namespace __sanitizer { void NORETURN internal__exit(int exitcode) { _zx_process_exit(exitcode); } uptr internal_sched_yield() { - zx_status_t status = _zx_nanosleep(0); + zx_status_t status = _zx_thread_legacy_yield(0u); CHECK_EQ(status, ZX_OK); return 0; // Why doesn't this return void? } @@ -87,7 +87,6 @@ void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) { } void InitializePlatformEarly() {} -void MaybeReexec() {} void CheckASLR() {} void CheckMPROTECT() {} void PlatformPrepareForSandboxing(void *args) {} @@ -128,6 +127,8 @@ uptr GetMaxUserVirtualAddress() { uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); } +bool ErrorIsOOM(error_t err) { return err == ZX_ERR_NO_MEMORY; } + static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type, bool raw_report, bool die_for_nomem) { size = RoundUpTo(size, GetPageSize()); diff --git a/libsanitizer/sanitizer_common/sanitizer_interface_internal.h b/libsanitizer/sanitizer_common/sanitizer_interface_internal.h index e9dc78c6354..ad34e5e5ba5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_interface_internal.h +++ b/libsanitizer/sanitizer_common/sanitizer_interface_internal.h @@ -107,6 +107,26 @@ __sanitizer_cov_trace_gep(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_indir(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load1(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load2(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load4(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load8(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load16(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store1(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store2(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store4(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store8(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store16(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_guard(__sanitizer::u32 *); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32 *, __sanitizer::u32 *); diff --git a/libsanitizer/sanitizer_common/sanitizer_internal_defs.h b/libsanitizer/sanitizer_common/sanitizer_internal_defs.h index ff65069de8d..6b800820ab8 100644 --- a/libsanitizer/sanitizer_common/sanitizer_internal_defs.h +++ b/libsanitizer/sanitizer_common/sanitizer_internal_defs.h @@ -73,7 +73,7 @@ // Before Xcode 4.5, the Darwin linker doesn't reliably support undefined // weak symbols. Mac OS X 10.9/Darwin 13 is the first release only supported // by Xcode >= 4.5. -#elif SANITIZER_MAC && \ +#elif SANITIZER_APPLE && \ __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1090 && !SANITIZER_GO # define SANITIZER_SUPPORTS_WEAK_HOOKS 1 #else @@ -139,7 +139,7 @@ namespace __sanitizer { typedef unsigned long long uptr; typedef signed long long sptr; #else -# if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC || SANITIZER_WINDOWS +# if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE || SANITIZER_WINDOWS typedef unsigned long uptr; typedef signed long sptr; # else @@ -177,7 +177,7 @@ typedef long pid_t; typedef int pid_t; #endif -#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC || \ +#if SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE || \ (SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \ (SANITIZER_LINUX && !SANITIZER_GLIBC && !SANITIZER_ANDROID) || \ (SANITIZER_LINUX && (defined(__x86_64__) || defined(__hexagon__))) @@ -187,7 +187,7 @@ typedef uptr OFF_T; #endif typedef u64 OFF64_T; -#if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC +#if (SANITIZER_WORDSIZE == 64) || SANITIZER_APPLE typedef uptr operator_new_size_type; #else # if defined(__s390__) && !defined(__s390x__) @@ -386,13 +386,10 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, enum LinkerInitialized { LINKER_INITIALIZED = 0 }; #if !defined(_MSC_VER) || defined(__clang__) -#if SANITIZER_S390_31 -#define GET_CALLER_PC() \ - (__sanitizer::uptr) __builtin_extract_return_addr(__builtin_return_address(0)) -#else -#define GET_CALLER_PC() (__sanitizer::uptr) __builtin_return_address(0) -#endif -#define GET_CURRENT_FRAME() (__sanitizer::uptr) __builtin_frame_address(0) +# define GET_CALLER_PC() \ + ((__sanitizer::uptr)__builtin_extract_return_addr( \ + __builtin_return_address(0))) +# define GET_CURRENT_FRAME() ((__sanitizer::uptr)__builtin_frame_address(0)) inline void Trap() { __builtin_trap(); } @@ -401,13 +398,13 @@ extern "C" void* _ReturnAddress(void); extern "C" void* _AddressOfReturnAddress(void); # pragma intrinsic(_ReturnAddress) # pragma intrinsic(_AddressOfReturnAddress) -#define GET_CALLER_PC() (__sanitizer::uptr) _ReturnAddress() +# define GET_CALLER_PC() ((__sanitizer::uptr)_ReturnAddress()) // CaptureStackBackTrace doesn't need to know BP on Windows. -#define GET_CURRENT_FRAME() \ - (((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr)) +# define GET_CURRENT_FRAME() \ + (((__sanitizer::uptr)_AddressOfReturnAddress()) + sizeof(__sanitizer::uptr)) extern "C" void __ud2(void); -# pragma intrinsic(__ud2) +# pragma intrinsic(__ud2) inline void Trap() { __ud2(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_libignore.cpp b/libsanitizer/sanitizer_common/sanitizer_libignore.cpp index caaba3155a7..b7fc9444cc6 100644 --- a/libsanitizer/sanitizer_common/sanitizer_libignore.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_libignore.cpp @@ -8,7 +8,7 @@ #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \ +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_APPLE || \ SANITIZER_NETBSD #include "sanitizer_libignore.h" @@ -125,5 +125,5 @@ void LibIgnore::OnLibraryUnloaded() { } // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_APPLE || // SANITIZER_NETBSD diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_linux.cpp index 5ba033492e7..dc2ea933fad 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_linux.cpp @@ -78,6 +78,10 @@ #include <sys/personality.h> #endif +#if SANITIZER_LINUX && defined(__loongarch__) +# include <sys/sysmacros.h> +#endif + #if SANITIZER_FREEBSD #include <sys/exec.h> #include <sys/procctl.h> @@ -188,6 +192,8 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); } # include "sanitizer_syscall_linux_arm.inc" # elif SANITIZER_LINUX && defined(__hexagon__) # include "sanitizer_syscall_linux_hexagon.inc" +# elif SANITIZER_LINUX && SANITIZER_LOONGARCH64 +# include "sanitizer_syscall_linux_loongarch64.inc" # else # include "sanitizer_syscall_generic.inc" # endif @@ -271,7 +277,7 @@ uptr internal_ftruncate(fd_t fd, uptr size) { return res; } -#if !SANITIZER_LINUX_USES_64BIT_SYSCALLS && SANITIZER_LINUX +#if (!SANITIZER_LINUX_USES_64BIT_SYSCALLS || SANITIZER_SPARC) && SANITIZER_LINUX static void stat64_to_stat(struct stat64 *in, struct stat *out) { internal_memset(out, 0, sizeof(*out)); out->st_dev = in->st_dev; @@ -290,6 +296,28 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) { } #endif +#if SANITIZER_LINUX && defined(__loongarch__) +static void statx_to_stat(struct statx *in, struct stat *out) { + internal_memset(out, 0, sizeof(*out)); + out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor); + out->st_ino = in->stx_ino; + out->st_mode = in->stx_mode; + out->st_nlink = in->stx_nlink; + out->st_uid = in->stx_uid; + out->st_gid = in->stx_gid; + out->st_rdev = makedev(in->stx_rdev_major, in->stx_rdev_minor); + out->st_size = in->stx_size; + out->st_blksize = in->stx_blksize; + out->st_blocks = in->stx_blocks; + out->st_atime = in->stx_atime.tv_sec; + out->st_atim.tv_nsec = in->stx_atime.tv_nsec; + out->st_mtime = in->stx_mtime.tv_sec; + out->st_mtim.tv_nsec = in->stx_mtime.tv_nsec; + out->st_ctime = in->stx_ctime.tv_sec; + out->st_ctim.tv_nsec = in->stx_ctime.tv_nsec; +} +#endif + #if SANITIZER_MIPS64 // Undefine compatibility macros from <sys/stat.h> // so that they would not clash with the kernel_stat @@ -341,50 +369,65 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { #endif uptr internal_stat(const char *path, void *buf) { -#if SANITIZER_FREEBSD +# if SANITIZER_FREEBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); -# elif SANITIZER_LINUX -# if SANITIZER_WORDSIZE == 64 || SANITIZER_X32 || \ - (defined(__mips__) && _MIPS_SIM == _ABIN32) +# elif SANITIZER_LINUX +# if defined(__loongarch__) + struct statx bufx; + int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path, + AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx); + statx_to_stat(&bufx, (struct stat *)buf); + return res; +# elif (SANITIZER_WORDSIZE == 64 || SANITIZER_X32 || \ + (defined(__mips__) && _MIPS_SIM == _ABIN32)) && \ + !SANITIZER_SPARC return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); -# else +# else struct stat64 buf64; int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path, (uptr)&buf64, 0); stat64_to_stat(&buf64, (struct stat *)buf); return res; -# endif -# else +# endif +# else struct stat64 buf64; int res = internal_syscall(SYSCALL(stat64), path, &buf64); stat64_to_stat(&buf64, (struct stat *)buf); return res; -# endif +# endif } uptr internal_lstat(const char *path, void *buf) { -#if SANITIZER_FREEBSD +# if SANITIZER_FREEBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); -# elif SANITIZER_LINUX -# if defined(_LP64) || SANITIZER_X32 || \ - (defined(__mips__) && _MIPS_SIM == _ABIN32) +# elif SANITIZER_LINUX +# if defined(__loongarch__) + struct statx bufx; + int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path, + AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT, + STATX_BASIC_STATS, (uptr)&bufx); + statx_to_stat(&bufx, (struct stat *)buf); + return res; +# elif (defined(_LP64) || SANITIZER_X32 || \ + (defined(__mips__) && _MIPS_SIM == _ABIN32)) && \ + !SANITIZER_SPARC return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); -# else +# else struct stat64 buf64; int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path, (uptr)&buf64, AT_SYMLINK_NOFOLLOW); stat64_to_stat(&buf64, (struct stat *)buf); return res; -# endif -# else +# endif +# else struct stat64 buf64; int res = internal_syscall(SYSCALL(lstat64), path, &buf64); stat64_to_stat(&buf64, (struct stat *)buf); return res; -# endif +# endif } uptr internal_fstat(fd_t fd, void *buf) { @@ -395,9 +438,15 @@ uptr internal_fstat(fd_t fd, void *buf) { int res = internal_syscall(SYSCALL(fstat), fd, &kbuf); kernel_stat_to_stat(&kbuf, (struct stat *)buf); return res; -# else +# elif SANITIZER_LINUX && defined(__loongarch__) + struct statx bufx; + int res = internal_syscall(SYSCALL(statx), fd, 0, AT_EMPTY_PATH, + STATX_BASIC_STATS, (uptr)&bufx); + statx_to_stat(&bufx, (struct stat *)buf); + return res; +# else return internal_syscall(SYSCALL(fstat), fd, (uptr)buf); -# endif +# endif #else struct stat64 buf64; int res = internal_syscall(SYSCALL(fstat64), fd, &buf64); @@ -443,15 +492,15 @@ uptr internal_unlink(const char *path) { } uptr internal_rename(const char *oldpath, const char *newpath) { -#if defined(__riscv) && defined(__linux__) +# if (defined(__riscv) || defined(__loongarch__)) && defined(__linux__) return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD, (uptr)newpath, 0); -# elif SANITIZER_LINUX +# elif SANITIZER_LINUX return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD, (uptr)newpath); -# else +# else return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); -# endif +# endif } uptr internal_sched_yield() { @@ -761,7 +810,14 @@ uptr internal_lseek(fd_t fd, OFF_T offset, int whence) { uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5); } -#endif +# if defined(__x86_64__) +# include <asm/unistd_64.h> +// Currently internal_arch_prctl() is only needed on x86_64. +uptr internal_arch_prctl(int option, uptr arg2) { + return internal_syscall(__NR_arch_prctl, option, arg2); +} +# endif +# endif uptr internal_sigaltstack(const void *ss, void *oss) { return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); @@ -904,6 +960,10 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { return k_set->sig[idx] & ((uptr)1 << bit); } #elif SANITIZER_FREEBSD +uptr internal_procctl(int type, int id, int cmd, void *data) { + return internal_syscall(SYSCALL(procctl), type, id, cmd, data); +} + void internal_sigdelset(__sanitizer_sigset_t *set, int signum) { sigset_t *rset = reinterpret_cast<sigset_t *>(set); sigdelset(rset, signum); @@ -1792,7 +1852,7 @@ void *internal_start_thread(void *(*func)(void *), void *arg) { return 0; } void internal_join_thread(void *th) {} #endif -#if defined(__aarch64__) +#if SANITIZER_LINUX && defined(__aarch64__) // Android headers in the older NDK releases miss this definition. struct __sanitizer_esr_context { struct _aarch64_ctx head; @@ -1813,6 +1873,11 @@ static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) { } return false; } +#elif SANITIZER_FREEBSD && defined(__aarch64__) +// FreeBSD doesn't provide ESR in the ucontext. +static bool Aarch64GetESR(ucontext_t *ucontext, u64 *esr) { + return false; +} #endif using Context = ucontext_t; @@ -2038,10 +2103,17 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *bp = ucontext->uc_mcontext.arm_fp; *sp = ucontext->uc_mcontext.arm_sp; #elif defined(__aarch64__) +# if SANITIZER_FREEBSD + ucontext_t *ucontext = (ucontext_t*)context; + *pc = ucontext->uc_mcontext.mc_gpregs.gp_elr; + *bp = ucontext->uc_mcontext.mc_gpregs.gp_x[29]; + *sp = ucontext->uc_mcontext.mc_gpregs.gp_sp; +# else ucontext_t *ucontext = (ucontext_t*)context; *pc = ucontext->uc_mcontext.pc; *bp = ucontext->uc_mcontext.regs[29]; *sp = ucontext->uc_mcontext.sp; +# endif #elif defined(__hppa__) ucontext_t *ucontext = (ucontext_t*)context; *pc = ucontext->uc_mcontext.sc_iaoq[0]; @@ -2151,6 +2223,11 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = ucontext->uc_mcontext.pc; *bp = ucontext->uc_mcontext.r30; *sp = ucontext->uc_mcontext.r29; +# elif defined(__loongarch__) + ucontext_t *ucontext = (ucontext_t *)context; + *pc = ucontext->uc_mcontext.__pc; + *bp = ucontext->uc_mcontext.__gregs[22]; + *sp = ucontext->uc_mcontext.__gregs[3]; # else # error "Unsupported arch" # endif @@ -2162,10 +2239,6 @@ void InitializePlatformEarly() { // Do nothing. } -void MaybeReexec() { - // No need to re-exec on Linux. -} - void CheckASLR() { #if SANITIZER_NETBSD int mib[3]; @@ -2189,7 +2262,8 @@ void CheckASLR() { } #elif SANITIZER_FREEBSD int aslr_status; - if (UNLIKELY(procctl(P_PID, 0, PROC_ASLR_STATUS, &aslr_status) == -1)) { + int r = internal_procctl(P_PID, 0, PROC_ASLR_STATUS, &aslr_status); + if (UNLIKELY(r == -1)) { // We're making things less 'dramatic' here since // the cmd is not necessarily guaranteed to be here // just yet regarding FreeBSD release diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.h b/libsanitizer/sanitizer_common/sanitizer_linux.h index ebd60e0b10f..761c57d1b8e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux.h +++ b/libsanitizer/sanitizer_common/sanitizer_linux.h @@ -69,6 +69,9 @@ uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp); // Linux-only syscalls. #if SANITIZER_LINUX uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5); +# if defined(__x86_64__) +uptr internal_arch_prctl(int option, uptr arg2); +# endif // Used only by sanitizer_stoptheworld. Signal handlers that are actually used // (like the process-wide error reporting SEGV handler) must use // internal_sigaction instead. @@ -82,6 +85,7 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, #endif int internal_uname(struct utsname *buf); #elif SANITIZER_FREEBSD +uptr internal_procctl(int type, int id, int cmd, void *data); void internal_sigdelset(__sanitizer_sigset_t *set, int signum); #elif SANITIZER_NETBSD void internal_sigdelset(__sanitizer_sigset_t *set, int signum); diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp index 6dd27402ac9..d74851c43e1 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -421,14 +421,14 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size, return; } // Find the maximum consecutive ranges. We consider two modules consecutive if - // the gap is smaller than the alignment. The dynamic loader places static TLS - // blocks this way not to waste space. + // the gap is smaller than the alignment of the latter range. The dynamic + // loader places static TLS blocks this way not to waste space. uptr l = one; *align = ranges[l].align; - while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l - 1].align) + while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align) *align = Max(*align, ranges[--l].align); uptr r = one + 1; - while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r - 1].align) + while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align) *align = Max(*align, ranges[r++].align); *addr = ranges[l].begin; *size = ranges[r - 1].end - ranges[l].begin; @@ -822,13 +822,9 @@ u32 GetNumberOfCPUs() { #elif SANITIZER_SOLARIS return sysconf(_SC_NPROCESSORS_ONLN); #else -#if defined(CPU_COUNT) cpu_set_t CPUs; CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); return CPU_COUNT(&CPUs); -#else - return 1; -#endif #endif } diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_mac.cpp index 7ce8d670ecc..1ae69e14b23 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_mac.cpp @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "sanitizer_mac.h" #include "interception/interception.h" @@ -38,7 +38,7 @@ extern char **environ; #endif -#if defined(__has_include) && __has_include(<os/trace.h>) && defined(__BLOCKS__) +#if defined(__has_include) && __has_include(<os/trace.h>) #define SANITIZER_OS_TRACE 1 #include <os/trace.h> #else @@ -71,16 +71,9 @@ extern "C" { #include <mach/mach_time.h> #include <mach/vm_statistics.h> #include <malloc/malloc.h> -#if defined(__has_builtin) && __has_builtin(__builtin_os_log_format) -# include <os/log.h> -#else - /* Without support for __builtin_os_log_format, fall back to the older - method. */ -# define OS_LOG_DEFAULT 0 -# define os_log_error(A,B,C) \ - asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", (C)); -#endif +#include <os/log.h> #include <pthread.h> +#include <pthread/introspection.h> #include <sched.h> #include <signal.h> #include <spawn.h> @@ -888,6 +881,9 @@ SignalContext::WriteFlag SignalContext::GetWriteFlag() const { #if defined(__x86_64__) || defined(__i386__) ucontext_t *ucontext = static_cast<ucontext_t*>(context); return ucontext->uc_mcontext->__es.__err & 2 /*T_PF_WRITE*/ ? Write : Read; +#elif defined(__arm64__) + ucontext_t *ucontext = static_cast<ucontext_t*>(context); + return ucontext->uc_mcontext->__es.__esr & 0x40 /*ISS_DA_WNR*/ ? Write : Read; #else return Unknown; #endif @@ -948,6 +944,9 @@ static void DisableMmapExcGuardExceptions() { set_behavior(mach_task_self(), task_exc_guard_none); } +static void VerifyInterceptorsWorking(); +static void StripEnv(); + void InitializePlatformEarly() { // Only use xnu_fast_mmap when on x86_64 and the kernel supports it. use_xnu_fast_mmap = @@ -958,17 +957,54 @@ void InitializePlatformEarly() { #endif if (GetDarwinKernelVersion() >= DarwinKernelVersion(19, 0)) DisableMmapExcGuardExceptions(); + +# if !SANITIZER_GO + MonotonicNanoTime(); // Call to initialize mach_timebase_info + VerifyInterceptorsWorking(); + StripEnv(); +# endif } #if !SANITIZER_GO static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; LowLevelAllocator allocator_for_env; +static bool ShouldCheckInterceptors() { + // Restrict "interceptors working?" check to ASan and TSan. + const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"}; + size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]); + for (size_t i = 0; i < count; i++) { + if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0) + return true; + } + return false; +} + +static void VerifyInterceptorsWorking() { + if (!common_flags()->verify_interceptors || !ShouldCheckInterceptors()) + return; + + // Verify that interceptors really work. We'll use dlsym to locate + // "puts", if interceptors are working, it should really point to + // "wrap_puts" within our own dylib. + Dl_info info_puts, info_runtime; + RAW_CHECK(dladdr(dlsym(RTLD_DEFAULT, "puts"), &info_puts)); + RAW_CHECK(dladdr((void *)__sanitizer_report_error_summary, &info_runtime)); + if (internal_strcmp(info_puts.dli_fname, info_runtime.dli_fname) != 0) { + Report( + "ERROR: Interceptors are not working. This may be because %s is " + "loaded too late (e.g. via dlopen). Please launch the executable " + "with:\n%s=%s\n", + SanitizerToolName, kDyldInsertLibraries, info_runtime.dli_fname); + RAW_CHECK("interceptors not installed" && 0); + } +} + // Change the value of the env var |name|, leaking the original value. // If |name_value| is NULL, the variable is deleted from the environment, // otherwise the corresponding "NAME=value" string is replaced with // |name_value|. -void LeakyResetEnv(const char *name, const char *name_value) { +static void LeakyResetEnv(const char *name, const char *name_value) { char **env = GetEnviron(); uptr name_len = internal_strlen(name); while (*env != 0) { @@ -993,100 +1029,28 @@ void LeakyResetEnv(const char *name, const char *name_value) { } } -SANITIZER_WEAK_CXX_DEFAULT_IMPL -bool ReexecDisabled() { - return false; -} - -static bool DyldNeedsEnvVariable() { - // If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if - // DYLD_INSERT_LIBRARIES is not set. - return GetMacosAlignedVersion() < MacosVersion(10, 11); -} - -void MaybeReexec() { - // FIXME: This should really live in some "InitializePlatform" method. - MonotonicNanoTime(); +static void StripEnv() { + if (!common_flags()->strip_env) + return; - if (ReexecDisabled()) return; + char *dyld_insert_libraries = + const_cast<char *>(GetEnv(kDyldInsertLibraries)); + if (!dyld_insert_libraries) + return; - // Make sure the dynamic runtime library is preloaded so that the - // wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec - // ourselves. Dl_info info; - RAW_CHECK(dladdr((void*)((uptr)&__sanitizer_report_error_summary), &info)); - char *dyld_insert_libraries = - const_cast<char*>(GetEnv(kDyldInsertLibraries)); - uptr old_env_len = dyld_insert_libraries ? - internal_strlen(dyld_insert_libraries) : 0; - uptr fname_len = internal_strlen(info.dli_fname); + RAW_CHECK(dladdr((void *)__sanitizer_report_error_summary, &info)); const char *dylib_name = StripModuleName(info.dli_fname); - uptr dylib_name_len = internal_strlen(dylib_name); - - bool lib_is_in_env = dyld_insert_libraries && - internal_strstr(dyld_insert_libraries, dylib_name); - if (DyldNeedsEnvVariable() && !lib_is_in_env) { - // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime - // library. - InternalMmapVector<char> program_name(1024); - uint32_t buf_size = program_name.size(); - _NSGetExecutablePath(program_name.data(), &buf_size); - char *new_env = const_cast<char*>(info.dli_fname); - if (dyld_insert_libraries) { - // Append the runtime dylib name to the existing value of - // DYLD_INSERT_LIBRARIES. - new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2); - internal_strncpy(new_env, dyld_insert_libraries, old_env_len); - new_env[old_env_len] = ':'; - // Copy fname_len and add a trailing zero. - internal_strncpy(new_env + old_env_len + 1, info.dli_fname, - fname_len + 1); - // Ok to use setenv() since the wrappers don't depend on the value of - // asan_inited. - setenv(kDyldInsertLibraries, new_env, /*overwrite*/1); - } else { - // Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name. - setenv(kDyldInsertLibraries, info.dli_fname, /*overwrite*/0); - } - VReport(1, "exec()-ing the program with\n"); - VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env); - VReport(1, "to enable wrappers.\n"); - execv(program_name.data(), *_NSGetArgv()); - - // We get here only if execv() failed. - Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, " - "which is required for the sanitizer to work. We tried to set the " - "environment variable and re-execute itself, but execv() failed, " - "possibly because of sandbox restrictions. Make sure to launch the " - "executable with:\n%s=%s\n", kDyldInsertLibraries, new_env); - RAW_CHECK("execv failed" && 0); - } - - // Verify that interceptors really work. We'll use dlsym to locate - // "puts", if interceptors are working, it should really point to - // "wrap_puts" within our own dylib. - Dl_info info_puts; - void *dlopen_addr = dlsym(RTLD_DEFAULT, "puts"); - RAW_CHECK(dladdr(dlopen_addr, &info_puts)); - if (internal_strcmp(info.dli_fname, info_puts.dli_fname) != 0) { - Report( - "ERROR: Interceptors are not working. This may be because %s is " - "loaded too late (e.g. via dlopen). Please launch the executable " - "with:\n%s=%s\n", - SanitizerToolName, kDyldInsertLibraries, info.dli_fname); - RAW_CHECK("interceptors not installed" && 0); - } - + bool lib_is_in_env = internal_strstr(dyld_insert_libraries, dylib_name); if (!lib_is_in_env) return; - if (!common_flags()->strip_env) - return; - // DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove // the dylib from the environment variable, because interceptors are installed // and we don't want our children to inherit the variable. + uptr old_env_len = internal_strlen(dyld_insert_libraries); + uptr dylib_name_len = internal_strlen(dylib_name); uptr env_name_len = internal_strlen(kDyldInsertLibraries); // Allocate memory to hold the previous env var name, its value, the '=' // sign and the '\0' char. @@ -1432,6 +1396,61 @@ u32 GetNumberOfCPUs() { void InitializePlatformCommonFlags(CommonFlags *cf) {} +// Pthread introspection hook +// +// * GCD worker threads are created without a call to pthread_create(), but we +// still need to register these threads (with ThreadCreate/Start()). +// * We use the "pthread introspection hook" below to observe the creation of +// such threads. +// * GCD worker threads don't have parent threads and the CREATE event is +// delivered in the context of the thread itself. CREATE events for regular +// threads, are delivered on the parent. We use this to tell apart which +// threads are GCD workers with `thread == pthread_self()`. +// +static pthread_introspection_hook_t prev_pthread_introspection_hook; +static ThreadEventCallbacks thread_event_callbacks; + +static void sanitizer_pthread_introspection_hook(unsigned int event, + pthread_t thread, void *addr, + size_t size) { + // create -> start -> terminate -> destroy + // * create/destroy are usually (not guaranteed) delivered on the parent and + // track resource allocation/reclamation + // * start/terminate are guaranteed to be delivered in the context of the + // thread and give hooks into "just after (before) thread starts (stops) + // executing" + DCHECK(event >= PTHREAD_INTROSPECTION_THREAD_CREATE && + event <= PTHREAD_INTROSPECTION_THREAD_DESTROY); + + if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) { + bool gcd_worker = (thread == pthread_self()); + if (thread_event_callbacks.create) + thread_event_callbacks.create((uptr)thread, gcd_worker); + } else if (event == PTHREAD_INTROSPECTION_THREAD_START) { + CHECK_EQ(thread, pthread_self()); + if (thread_event_callbacks.start) + thread_event_callbacks.start((uptr)thread); + } + + if (prev_pthread_introspection_hook) + prev_pthread_introspection_hook(event, thread, addr, size); + + if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { + CHECK_EQ(thread, pthread_self()); + if (thread_event_callbacks.terminate) + thread_event_callbacks.terminate((uptr)thread); + } else if (event == PTHREAD_INTROSPECTION_THREAD_DESTROY) { + if (thread_event_callbacks.destroy) + thread_event_callbacks.destroy((uptr)thread); + } +} + +void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks) { + thread_event_callbacks = callbacks; + prev_pthread_introspection_hook = + pthread_introspection_hook_install(&sanitizer_pthread_introspection_hook); +} + } // namespace __sanitizer -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.h b/libsanitizer/sanitizer_common/sanitizer_mac.h index 96a5986a47a..f0a97d098ee 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac.h +++ b/libsanitizer/sanitizer_common/sanitizer_mac.h @@ -9,32 +9,12 @@ // This file is shared between various sanitizers' runtime libraries and // provides definitions for OSX-specific functions. //===----------------------------------------------------------------------===// -#ifndef SANITIZER_MAC_H -#define SANITIZER_MAC_H +#ifndef SANITIZER_APPLE_H +#define SANITIZER_APPLE_H #include "sanitizer_common.h" #include "sanitizer_platform.h" - -/* TARGET_OS_OSX is not present in SDKs before Darwin16 (macOS 10.12) use - TARGET_OS_MAC (we have no support for iOS in any form for these versions, - so there's no ambiguity). */ -#if !defined(TARGET_OS_OSX) && TARGET_OS_MAC -# define TARGET_OS_OSX 1 -#endif - -/* Other TARGET_OS_xxx are not present on earlier versions, define them to - 0 (we have no support for them; they are not valid targets anyway). */ -#ifndef TARGET_OS_IOS -#define TARGET_OS_IOS 0 -#endif -#ifndef TARGET_OS_TV -#define TARGET_OS_TV 0 -#endif -#ifndef TARGET_OS_WATCH -#define TARGET_OS_WATCH 0 -#endif - -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "sanitizer_posix.h" namespace __sanitizer { @@ -82,7 +62,18 @@ char **GetEnviron(); void RestrictMemoryToMaxAddress(uptr max_address); +using ThreadEventCallback = void (*)(uptr thread); +using ThreadCreateEventCallback = void (*)(uptr thread, bool gcd_worker); +struct ThreadEventCallbacks { + ThreadCreateEventCallback create; + ThreadEventCallback start; + ThreadEventCallback terminate; + ThreadEventCallback destroy; +}; + +void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks); + } // namespace __sanitizer -#endif // SANITIZER_MAC -#endif // SANITIZER_MAC_H +#endif // SANITIZER_APPLE +#endif // SANITIZER_APPLE_H diff --git a/libsanitizer/sanitizer_common/sanitizer_mac_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_mac_libcdep.cpp index ac7e328946b..b452dc4a49e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_mac_libcdep.cpp @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "sanitizer_mac.h" #include <sys/mman.h> @@ -26,4 +26,4 @@ void RestrictMemoryToMaxAddress(uptr max_address) { } // namespace __sanitizer -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc index 764e2cef5e7..fe76b3f8aa0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc +++ b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if !SANITIZER_MAC +#if !SANITIZER_APPLE #error "This file should only be compiled on Darwin." #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_mutex.h b/libsanitizer/sanitizer_common/sanitizer_mutex.h index d2188a9e6d6..b1a58e421d8 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mutex.h +++ b/libsanitizer/sanitizer_common/sanitizer_mutex.h @@ -101,7 +101,7 @@ enum { // THREADLOCAL variables they are not usable early on during process init when // `__sanitizer::Mutex` is used. #define SANITIZER_CHECK_DEADLOCKS \ - (SANITIZER_DEBUG && !SANITIZER_GO && SANITIZER_SUPPORTS_THREADLOCAL && !SANITIZER_MAC) + (SANITIZER_DEBUG && !SANITIZER_GO && SANITIZER_SUPPORTS_THREADLOCAL && !SANITIZER_APPLE) #if SANITIZER_CHECK_DEADLOCKS struct MutexMeta { diff --git a/libsanitizer/sanitizer_common/sanitizer_openbsd.cpp b/libsanitizer/sanitizer_common/sanitizer_openbsd.cpp deleted file mode 100644 index e69de29bb2d..00000000000 --- a/libsanitizer/sanitizer_common/sanitizer_openbsd.cpp +++ /dev/null diff --git a/libsanitizer/sanitizer_common/sanitizer_platform.h b/libsanitizer/sanitizer_common/sanitizer_platform.h index 8bd9a327623..32005eef08c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform.h @@ -55,8 +55,15 @@ # define SANITIZER_SOLARIS 0 #endif +// - SANITIZER_APPLE: all Apple code +// - TARGET_OS_OSX: macOS +// - SANITIZER_IOS: devices (iOS and iOS-like) +// - SANITIZER_WATCHOS +// - SANITIZER_TVOS +// - SANITIZER_IOSSIM: simulators (iOS and iOS-like) +// - SANITIZER_DRIVERKIT #if defined(__APPLE__) -# define SANITIZER_MAC 1 +# define SANITIZER_APPLE 1 # include <TargetConditionals.h> # if TARGET_OS_OSX # define SANITIZER_OSX 1 @@ -68,28 +75,34 @@ # else # define SANITIZER_IOS 0 # endif +# if TARGET_OS_WATCH +# define SANITIZER_WATCHOS 1 +# else +# define SANITIZER_WATCHOS 0 +# endif +# if TARGET_OS_TV +# define SANITIZER_TVOS 1 +# else +# define SANITIZER_TVOS 0 +# endif # if TARGET_OS_SIMULATOR # define SANITIZER_IOSSIM 1 # else # define SANITIZER_IOSSIM 0 # endif +# if defined(TARGET_OS_DRIVERKIT) && TARGET_OS_DRIVERKIT +# define SANITIZER_DRIVERKIT 1 +# else +# define SANITIZER_DRIVERKIT 0 +# endif #else -# define SANITIZER_MAC 0 -# define SANITIZER_IOS 0 -# define SANITIZER_IOSSIM 0 +# define SANITIZER_APPLE 0 # define SANITIZER_OSX 0 -#endif - -#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_WATCH -# define SANITIZER_WATCHOS 1 -#else +# define SANITIZER_IOS 0 # define SANITIZER_WATCHOS 0 -#endif - -#if defined(__APPLE__) && TARGET_OS_IPHONE && TARGET_OS_TV -# define SANITIZER_TVOS 1 -#else # define SANITIZER_TVOS 0 +# define SANITIZER_IOSSIM 0 +# define SANITIZER_DRIVERKIT 0 #endif #if defined(_WIN32) @@ -124,7 +137,7 @@ #endif #define SANITIZER_POSIX \ - (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \ + (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_APPLE || \ SANITIZER_NETBSD || SANITIZER_SOLARIS) #if __LP64__ || defined(_WIN64) @@ -187,6 +200,21 @@ # define SANITIZER_S390_64 0 #endif +#if defined(__sparc__) +# define SANITIZER_SPARC 1 +# if defined(__arch64__) +# define SANITIZER_SPARC32 0 +# define SANITIZER_SPARC64 1 +# else +# define SANITIZER_SPARC32 1 +# define SANITIZER_SPARC64 0 +# endif +#else +# define SANITIZER_SPARC 0 +# define SANITIZER_SPARC32 0 +# define SANITIZER_SPARC64 0 +#endif + #if defined(__powerpc__) # define SANITIZER_PPC 1 # if defined(__powerpc64__) @@ -244,6 +272,12 @@ # define SANITIZER_RISCV64 0 #endif +#if defined(__loongarch_lp64) +# define SANITIZER_LOONGARCH64 1 +#else +# define SANITIZER_LOONGARCH64 0 +#endif + // By default we allow to use SizeClassAllocator64 on 64-bit platform. // But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64 // does not work well and we need to fallback to SizeClassAllocator32. @@ -252,7 +286,8 @@ #ifndef SANITIZER_CAN_USE_ALLOCATOR64 # if (SANITIZER_ANDROID && defined(__aarch64__)) || SANITIZER_FUCHSIA # define SANITIZER_CAN_USE_ALLOCATOR64 1 -# elif defined(__mips64) || defined(__aarch64__) +# elif defined(__mips64) || defined(__aarch64__) || defined(__i386__) || \ + defined(__arm__) || SANITIZER_RISCV64 || defined(__hexagon__) # define SANITIZER_CAN_USE_ALLOCATOR64 0 # else # define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64) @@ -271,7 +306,7 @@ #elif SANITIZER_RISCV64 # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38) #elif defined(__aarch64__) -# if SANITIZER_MAC +# if SANITIZER_APPLE # if SANITIZER_OSX || SANITIZER_IOSSIM # define SANITIZER_MMAP_RANGE_SIZE \ FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) @@ -328,7 +363,7 @@ # define MSC_PREREQ(version) 0 #endif -#if SANITIZER_MAC && defined(__x86_64__) +#if SANITIZER_APPLE && defined(__x86_64__) # define SANITIZER_NON_UNIQUE_TYPEINFO 0 #else # define SANITIZER_NON_UNIQUE_TYPEINFO 1 @@ -356,7 +391,7 @@ # define SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT 0 #endif -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD || SANITIZER_SOLARIS +#if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD || SANITIZER_SOLARIS # define SANITIZER_MADVISE_DONTNEED MADV_FREE #else # define SANITIZER_MADVISE_DONTNEED MADV_DONTNEED @@ -380,7 +415,7 @@ // Enable ability to support sanitizer initialization that is // compatible with the sanitizer library being loaded via // `dlopen()`. -#if SANITIZER_MAC +#if SANITIZER_APPLE # define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1 #else # define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0 diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h index 3cbbead4e98..6e3081ec1fc 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h @@ -76,7 +76,7 @@ #define SI_LINUX 0 #endif -#if SANITIZER_MAC +#if SANITIZER_APPLE #define SI_MAC 1 #define SI_NOT_MAC 0 #else @@ -126,7 +126,7 @@ #define SI_SOLARIS32 0 #endif -#if SANITIZER_POSIX && !SANITIZER_MAC +#if SANITIZER_POSIX && !SANITIZER_APPLE #define SI_POSIX_NOT_MAC 1 #else #define SI_POSIX_NOT_MAC 0 @@ -236,6 +236,7 @@ #define SANITIZER_INTERCEPT_GLOB (SI_GLIBC || SI_SOLARIS) #define SANITIZER_INTERCEPT_GLOB64 SI_GLIBC #define SANITIZER_INTERCEPT___B64_TO SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_DN_COMP_EXPAND SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_POSIX_SPAWN SI_POSIX #define SANITIZER_INTERCEPT_WAIT SI_POSIX #define SANITIZER_INTERCEPT_INET SI_POSIX @@ -329,8 +330,7 @@ #define SANITIZER_INTERCEPT_GETMNTENT_R SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STATFS \ (SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) -#define SANITIZER_INTERCEPT_STATFS64 \ - (((SI_MAC && !TARGET_CPU_ARM64) && !SI_IOS) || SI_LINUX_NOT_ANDROID) +#define SANITIZER_INTERCEPT_STATFS64 SI_LINUX_NOT_ANDROID && SANITIZER_HAS_STATFS64 #define SANITIZER_INTERCEPT_STATVFS \ (SI_FREEBSD || SI_NETBSD || SI_LINUX_NOT_ANDROID) #define SANITIZER_INTERCEPT_STATVFS64 SI_LINUX_NOT_ANDROID @@ -347,6 +347,7 @@ #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_GLIBC +#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \ (SI_POSIX && !SI_NETBSD) @@ -466,7 +467,7 @@ #define SANITIZER_INTERCEPT_STAT \ (SI_FREEBSD || SI_MAC || SI_ANDROID || SI_NETBSD || SI_SOLARIS || \ SI_STAT_LINUX) -#define SANITIZER_INTERCEPT_STAT64 SI_STAT_LINUX +#define SANITIZER_INTERCEPT_STAT64 SI_STAT_LINUX && SANITIZER_HAS_STAT64 #define SANITIZER_INTERCEPT_LSTAT (SI_NETBSD || SI_FREEBSD || SI_STAT_LINUX) #define SANITIZER_INTERCEPT___XSTAT \ ((!SANITIZER_INTERCEPT_STAT && SI_POSIX) || SI_STAT_LINUX) @@ -587,10 +588,11 @@ // sigaltstack on i386 macOS cannot be intercepted due to setjmp() // calling it and assuming that it does not clobber registers. #define SANITIZER_INTERCEPT_SIGALTSTACK \ - (SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386)) + (SI_POSIX && !(SANITIZER_APPLE && SANITIZER_I386)) #define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD) #define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD #define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD +#define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD // This macro gives a way for downstream users to override the above // interceptor macros irrespective of the platform they are on. They have diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.cpp b/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.cpp index 0d25fa80e2e..37e72cd5d45 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.cpp @@ -205,6 +205,10 @@ unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); +unsigned struct_procctl_reaper_status_sz = sizeof(struct __sanitizer_procctl_reaper_status); +unsigned struct_procctl_reaper_pidinfo_sz = sizeof(struct __sanitizer_procctl_reaper_pidinfo); +unsigned struct_procctl_reaper_pids_sz = sizeof(struct __sanitizer_procctl_reaper_pids); +unsigned struct_procctl_reaper_kill_sz = sizeof(struct __sanitizer_procctl_reaper_kill); const unsigned long __sanitizer_bufsiz = BUFSIZ; const unsigned IOCTL_NOT_PRESENT = 0; diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.h index 9859c52ec69..daef1177a2d 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.h @@ -424,6 +424,38 @@ struct __sanitizer__ttyent { char *ty_group; }; +// procctl reaper data for PROCCTL_REAPER flags +struct __sanitizer_procctl_reaper_status { + unsigned int rs_flags; + unsigned int rs_children; + unsigned int rs_descendants; + pid_t rs_reaper; + pid_t rs_pid; + unsigned int rs_pad0[15]; +}; + +struct __sanitizer_procctl_reaper_pidinfo { + pid_t pi_pid; + pid_t pi_subtree; + unsigned int pi_flags; + unsigned int pi_pad0[15]; +}; + +struct __sanitizer_procctl_reaper_pids { + unsigned int rp_count; + unsigned int rp_pad0[15]; + struct __sanitize_procctl_reapper_pidinfo *rp_pids; +}; + +struct __sanitizer_procctl_reaper_kill { + int rk_sig; + unsigned int rk_flags; + pid_t rk_subtree; + unsigned int rk_killed; + pid_t rk_fpid; + unsigned int rk_pad[15]; +}; + # define IOC_NRBITS 8 # define IOC_TYPEBITS 8 # if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) @@ -480,6 +512,11 @@ extern unsigned struct_ppp_stats_sz; extern unsigned struct_sioc_sg_req_sz; extern unsigned struct_sioc_vif_req_sz; +extern unsigned struct_procctl_reaper_status_sz; +extern unsigned struct_procctl_reaper_pidinfo_sz; +extern unsigned struct_procctl_reaper_pids_sz; +extern unsigned struct_procctl_reaper_kill_sz; + // ioctl request identifiers // A special value to mark ioctls that are not present on the target platform, diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp index 2b1a2f7932c..bf0f355847c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp @@ -26,10 +26,7 @@ // With old kernels (and even new kernels on powerpc) asm/stat.h uses types that // are not defined anywhere in userspace headers. Fake them. This seems to work -// fine with newer headers, too. Beware that with <sys/stat.h>, struct stat -// takes the form of struct stat64 on 32-bit platforms if _FILE_OFFSET_BITS=64. -// Also, for some platforms (e.g. mips) there are additional members in the -// <sys/stat.h> struct stat:s. +// fine with newer headers, too. #include <linux/posix_types.h> # if defined(__x86_64__) || defined(__mips__) || defined(__hexagon__) # include <sys/stat.h> @@ -60,15 +57,10 @@ using namespace __sanitizer; -namespace __sanitizer { -#if !SANITIZER_ANDROID - unsigned struct_statfs64_sz = sizeof(struct statfs64); -#endif -} // namespace __sanitizer - # if !defined(__powerpc64__) && !defined(__x86_64__) && \ !defined(__aarch64__) && !defined(__mips__) && !defined(__s390__) && \ - !defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) + !defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) && \ + !defined(__loongarch__) COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat)); #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp index 97fd07acf9d..c85cf1626a7 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp @@ -23,7 +23,7 @@ // Must go after undef _FILE_OFFSET_BITS. #include "sanitizer_platform.h" -#if SANITIZER_LINUX || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_APPLE // Must go after undef _FILE_OFFSET_BITS. #include "sanitizer_glibc_version.h" @@ -51,7 +51,7 @@ #include <time.h> #include <wchar.h> #include <regex.h> -#if !SANITIZER_MAC +#if !SANITIZER_APPLE #include <utmp.h> #endif @@ -154,7 +154,6 @@ typedef struct user_fpregs elf_fpregset_t; #include <linux/serial.h> #include <sys/msg.h> #include <sys/ipc.h> -#include <crypt.h> #endif // SANITIZER_ANDROID #include <link.h> @@ -165,7 +164,7 @@ typedef struct user_fpregs elf_fpregset_t; #include <fstab.h> #endif // SANITIZER_LINUX -#if SANITIZER_MAC +#if SANITIZER_APPLE #include <net/ethernet.h> #include <sys/filio.h> #include <sys/sockio.h> @@ -174,14 +173,19 @@ typedef struct user_fpregs elf_fpregset_t; // Include these after system headers to avoid name clashes and ambiguities. # include "sanitizer_common.h" # include "sanitizer_internal_defs.h" +# include "sanitizer_platform_interceptors.h" # include "sanitizer_platform_limits_posix.h" +#if SANITIZER_INTERCEPT_CRYPT_R +#include <crypt.h> +#endif + namespace __sanitizer { unsigned struct_utsname_sz = sizeof(struct utsname); unsigned struct_stat_sz = sizeof(struct stat); -#if !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64) +#if SANITIZER_HAS_STAT64 unsigned struct_stat64_sz = sizeof(struct stat64); -#endif // !SANITIZER_IOS && !(SANITIZER_MAC && TARGET_CPU_ARM64) +#endif // SANITIZER_HAS_STAT64 unsigned struct_rusage_sz = sizeof(struct rusage); unsigned struct_tm_sz = sizeof(struct tm); unsigned struct_passwd_sz = sizeof(struct passwd); @@ -206,14 +210,14 @@ namespace __sanitizer { unsigned struct_regex_sz = sizeof(regex_t); unsigned struct_regmatch_sz = sizeof(regmatch_t); -#if (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS +#if SANITIZER_HAS_STATFS64 unsigned struct_statfs64_sz = sizeof(struct statfs64); -#endif // (SANITIZER_MAC && !TARGET_CPU_ARM64) && !SANITIZER_IOS +#endif // SANITIZER_HAS_STATFS64 -#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_MAC +#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD || SANITIZER_APPLE unsigned struct_fstab_sz = sizeof(struct fstab); #endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD || - // SANITIZER_MAC + // SANITIZER_APPLE #if !SANITIZER_ANDROID unsigned struct_statfs_sz = sizeof(struct statfs); unsigned struct_sockaddr_sz = sizeof(struct sockaddr); @@ -267,15 +271,22 @@ namespace __sanitizer { defined(__powerpc__) || defined(__s390__) || defined(__sparc__) || \ defined(__hexagon__) # define SIZEOF_STRUCT_USTAT 20 +# elif defined(__loongarch__) + // Not used. The minimum Glibc version available for LoongArch is 2.36 + // so ustat() wrapper is already gone. +# define SIZEOF_STRUCT_USTAT 0 # else # error Unknown size of struct ustat # endif unsigned struct_ustat_sz = SIZEOF_STRUCT_USTAT; unsigned struct_rlimit64_sz = sizeof(struct rlimit64); unsigned struct_statvfs64_sz = sizeof(struct statvfs64); - unsigned struct_crypt_data_sz = sizeof(struct crypt_data); #endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#if SANITIZER_INTERCEPT_CRYPT_R + unsigned struct_crypt_data_sz = sizeof(struct crypt_data); +#endif + #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_timex_sz = sizeof(struct timex); unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); @@ -302,7 +313,7 @@ namespace __sanitizer { int shmctl_shm_stat = (int)SHM_STAT; #endif -#if !SANITIZER_MAC && !SANITIZER_FREEBSD +#if !SANITIZER_APPLE && !SANITIZER_FREEBSD unsigned struct_utmp_sz = sizeof(struct utmp); #endif #if !SANITIZER_ANDROID @@ -510,7 +521,7 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); #endif // SANITIZER_GLIBC -#if !SANITIZER_ANDROID && !SANITIZER_MAC +#if !SANITIZER_ANDROID && !SANITIZER_APPLE unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); unsigned struct_sioc_vif_req_sz = sizeof(struct sioc_vif_req); #endif @@ -1069,7 +1080,7 @@ CHECK_SIZE_AND_OFFSET(mmsghdr, msg_len); COMPILER_CHECK(sizeof(__sanitizer_dirent) <= sizeof(dirent)); CHECK_SIZE_AND_OFFSET(dirent, d_ino); -#if SANITIZER_MAC +#if SANITIZER_APPLE CHECK_SIZE_AND_OFFSET(dirent, d_seekoff); #elif SANITIZER_FREEBSD // There is no 'd_off' field on FreeBSD. @@ -1251,7 +1262,7 @@ CHECK_SIZE_AND_OFFSET(passwd, pw_shell); CHECK_SIZE_AND_OFFSET(passwd, pw_gecos); #endif -#if SANITIZER_MAC +#if SANITIZER_APPLE CHECK_SIZE_AND_OFFSET(passwd, pw_change); CHECK_SIZE_AND_OFFSET(passwd, pw_expire); CHECK_SIZE_AND_OFFSET(passwd, pw_class); @@ -1264,7 +1275,7 @@ CHECK_SIZE_AND_OFFSET(group, gr_passwd); CHECK_SIZE_AND_OFFSET(group, gr_gid); CHECK_SIZE_AND_OFFSET(group, gr_mem); -#if HAVE_RPC_XDR_H +#if HAVE_RPC_XDR_H && !SANITIZER_APPLE CHECK_TYPE_SIZE(XDR); CHECK_SIZE_AND_OFFSET(XDR, x_op); CHECK_SIZE_AND_OFFSET(XDR, x_ops); @@ -1319,4 +1330,4 @@ CHECK_TYPE_SIZE(sem_t); COMPILER_CHECK(ARM_VFPREGS_SIZE == ARM_VFPREGS_SIZE_ASAN); #endif -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_APPLE diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h index 75c6cc7f285..bd5692ed511 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h @@ -14,11 +14,26 @@ #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H #define SANITIZER_PLATFORM_LIMITS_POSIX_H -#if SANITIZER_LINUX || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_APPLE #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" +#if SANITIZER_APPLE +#include <sys/cdefs.h> +#if !__DARWIN_ONLY_64_BIT_INO_T +#define SANITIZER_HAS_STAT64 1 +#define SANITIZER_HAS_STATFS64 1 +#else +#define SANITIZER_HAS_STAT64 0 +#define SANITIZER_HAS_STATFS64 0 +#endif +#else +// Must be SANITIZER_LINUX then +#define SANITIZER_HAS_STAT64 1 +#define SANITIZER_HAS_STATFS64 1 +#endif + #if defined(__sparc__) // FIXME: This can't be included from tsan which does not support sparc yet. #include "sanitizer_glibc_version.h" @@ -29,7 +44,7 @@ namespace __sanitizer { extern unsigned struct_utsname_sz; extern unsigned struct_stat_sz; -#if !SANITIZER_IOS +#if SANITIZER_HAS_STAT64 extern unsigned struct_stat64_sz; #endif extern unsigned struct_rusage_sz; @@ -49,7 +64,9 @@ extern unsigned struct_itimerspec_sz; extern unsigned struct_sigevent_sz; extern unsigned struct_stack_t_sz; extern unsigned struct_sched_param_sz; +#if SANITIZER_HAS_STATFS64 extern unsigned struct_statfs64_sz; +#endif extern unsigned struct_regex_sz; extern unsigned struct_regmatch_sz; @@ -106,6 +123,9 @@ const unsigned struct_kernel_stat64_sz = 0; // RISCV64 does not use stat64 # elif defined(__hexagon__) const unsigned struct_kernel_stat_sz = 128; const unsigned struct_kernel_stat64_sz = 0; +# elif defined(__loongarch__) +const unsigned struct_kernel_stat_sz = 128; +const unsigned struct_kernel_stat64_sz = 0; # endif struct __sanitizer_perf_event_attr { unsigned type; @@ -126,7 +146,7 @@ const unsigned struct_kexec_segment_sz = 4 * sizeof(unsigned long); #if SANITIZER_LINUX -#if defined(__powerpc64__) || defined(__s390__) +#if defined(__powerpc64__) || defined(__s390__) || defined(__loongarch__) const unsigned struct___old_kernel_stat_sz = 0; #elif !defined(__sparc__) const unsigned struct___old_kernel_stat_sz = 32; @@ -323,7 +343,7 @@ struct __sanitizer_ifaddrs { }; #endif // !SANITIZER_ANDROID -#if SANITIZER_MAC +#if SANITIZER_APPLE typedef unsigned long __sanitizer_pthread_key_t; #else typedef unsigned __sanitizer_pthread_key_t; @@ -350,7 +370,7 @@ struct __sanitizer_passwd { char *pw_passwd; int pw_uid; int pw_gid; -#if SANITIZER_MAC +#if SANITIZER_APPLE long pw_change; char *pw_class; #endif @@ -359,7 +379,7 @@ struct __sanitizer_passwd { #endif char *pw_dir; char *pw_shell; -#if SANITIZER_MAC +#if SANITIZER_APPLE long pw_expire; #endif }; @@ -432,7 +452,7 @@ struct __sanitizer_file_handle { }; #endif -#if SANITIZER_MAC +#if SANITIZER_APPLE struct __sanitizer_msghdr { void *msg_name; unsigned msg_namelen; @@ -473,7 +493,7 @@ struct __sanitizer_mmsghdr { }; #endif -#if SANITIZER_MAC +#if SANITIZER_APPLE struct __sanitizer_dirent { unsigned long long d_ino; unsigned long long d_seekoff; @@ -558,7 +578,7 @@ typedef unsigned long __sanitizer_sigset_t[16 / sizeof(unsigned long)]; # else typedef unsigned long __sanitizer_sigset_t; # endif -#elif SANITIZER_MAC +#elif SANITIZER_APPLE typedef unsigned __sanitizer_sigset_t; #elif SANITIZER_LINUX struct __sanitizer_sigset_t { @@ -730,7 +750,7 @@ struct __sanitizer_addrinfo { int ai_family; int ai_socktype; int ai_protocol; -#if SANITIZER_ANDROID || SANITIZER_MAC +#if SANITIZER_ANDROID || SANITIZER_APPLE unsigned ai_addrlen; char *ai_canonname; void *ai_addr; @@ -756,7 +776,7 @@ struct __sanitizer_pollfd { short revents; }; -#if SANITIZER_ANDROID || SANITIZER_MAC +#if SANITIZER_ANDROID || SANITIZER_APPLE typedef unsigned __sanitizer_nfds_t; #else typedef unsigned long __sanitizer_nfds_t; @@ -856,7 +876,7 @@ extern int shmctl_shm_info; extern int shmctl_shm_stat; #endif -#if !SANITIZER_MAC && !SANITIZER_FREEBSD +#if !SANITIZER_APPLE && !SANITIZER_FREEBSD extern unsigned struct_utmp_sz; #endif #if !SANITIZER_ANDROID @@ -871,7 +891,7 @@ struct __sanitizer_ifconf { union { void *ifcu_req; } ifc_ifcu; -#if SANITIZER_MAC +#if SANITIZER_APPLE } __attribute__((packed)); #else }; @@ -1024,7 +1044,7 @@ extern unsigned struct_audio_buf_info_sz; extern unsigned struct_ppp_stats_sz; #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID -#if !SANITIZER_ANDROID && !SANITIZER_MAC +#if !SANITIZER_ANDROID && !SANITIZER_APPLE extern unsigned struct_sioc_sg_req_sz; extern unsigned struct_sioc_vif_req_sz; #endif @@ -1465,6 +1485,6 @@ extern const int si_SEGV_ACCERR; #define SIGACTION_SYMNAME sigaction -#endif // SANITIZER_LINUX || SANITIZER_MAC +#endif // SANITIZER_LINUX || SANITIZER_APPLE #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_posix.cpp b/libsanitizer/sanitizer_common/sanitizer_posix.cpp index 3b330a3705e..b0e32b50c07 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_posix.cpp @@ -41,6 +41,8 @@ uptr GetMmapGranularity() { return GetPageSize(); } +bool ErrorIsOOM(error_t err) { return err == ENOMEM; } + void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { size = RoundUpTo(size, GetPageSizeCached()); uptr res = MmapNamed(nullptr, size, PROT_READ | PROT_WRITE, @@ -147,7 +149,7 @@ bool MprotectReadOnly(uptr addr, uptr size) { return 0 == internal_mprotect((void *)addr, size, PROT_READ); } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE void MprotectMallocZones(void *addr, int prot) {} #endif @@ -240,7 +242,7 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { return true; } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE void DumpProcessMap() { MemoryMappingLayout proc_maps(/*cache_enabled*/true); const sptr kBufSize = 4095; diff --git a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp index b6d8c7281bd..46e41c66973 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -384,7 +384,7 @@ real_pthread_attr_getstack(void *attr, void **addr, size_t *size); } // extern "C" int my_pthread_attr_getstack(void *attr, void **addr, uptr *size) { -#if !SANITIZER_GO && !SANITIZER_MAC +#if !SANITIZER_GO && !SANITIZER_APPLE if (&real_pthread_attr_getstack) return real_pthread_attr_getstack((pthread_attr_t *)attr, addr, (size_t *)size); diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps.h b/libsanitizer/sanitizer_common/sanitizer_procmaps.h index 055af366ef0..19bad158387 100644 --- a/libsanitizer/sanitizer_common/sanitizer_procmaps.h +++ b/libsanitizer/sanitizer_common/sanitizer_procmaps.h @@ -16,7 +16,7 @@ #include "sanitizer_platform.h" #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD || \ - SANITIZER_MAC || SANITIZER_SOLARIS || \ + SANITIZER_APPLE || SANITIZER_SOLARIS || \ SANITIZER_FUCHSIA #include "sanitizer_common.h" diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cpp index 62b2e5e0321..4b0e6781976 100644 --- a/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cpp @@ -10,7 +10,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "sanitizer_common.h" #include "sanitizer_placement_new.h" #include "sanitizer_procmaps.h" @@ -136,13 +136,19 @@ void MemoryMappingLayout::LoadFromCache() { // No-op on Mac for now. } +static bool IsDyldHdr(const mach_header *hdr) { + return (hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) && + hdr->filetype == MH_DYLINKER; +} + // _dyld_get_image_header() and related APIs don't report dyld itself. // We work around this by manually recursing through the memory map // until we hit a Mach header matching dyld instead. These recurse // calls are expensive, but the first memory map generation occurs // early in the process, when dyld is one of the only images loaded, -// so it will be hit after only a few iterations. -static mach_header *get_dyld_image_header() { +// so it will be hit after only a few iterations. These assumptions don't hold +// on macOS 13+ anymore (dyld itself has moved into the shared cache). +static mach_header *GetDyldImageHeaderViaVMRegion() { vm_address_t address = 0; while (true) { @@ -157,8 +163,7 @@ static mach_header *get_dyld_image_header() { if (size >= sizeof(mach_header) && info.protection & kProtectionRead) { mach_header *hdr = (mach_header *)address; - if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) && - hdr->filetype == MH_DYLINKER) { + if (IsDyldHdr(hdr)) { return hdr; } } @@ -166,8 +171,69 @@ static mach_header *get_dyld_image_header() { } } +extern "C" { +struct dyld_shared_cache_dylib_text_info { + uint64_t version; // current version 2 + // following fields all exist in version 1 + uint64_t loadAddressUnslid; + uint64_t textSegmentSize; + uuid_t dylibUuid; + const char *path; // pointer invalid at end of iterations + // following fields all exist in version 2 + uint64_t textSegmentOffset; // offset from start of cache +}; +typedef struct dyld_shared_cache_dylib_text_info + dyld_shared_cache_dylib_text_info; + +extern bool _dyld_get_shared_cache_uuid(uuid_t uuid); +extern const void *_dyld_get_shared_cache_range(size_t *length); +extern int dyld_shared_cache_iterate_text( + const uuid_t cacheUuid, + void (^callback)(const dyld_shared_cache_dylib_text_info *info)); +} // extern "C" + +static mach_header *GetDyldImageHeaderViaSharedCache() { + uuid_t uuid; + bool hasCache = _dyld_get_shared_cache_uuid(uuid); + if (!hasCache) + return nullptr; + + size_t cacheLength; + __block uptr cacheStart = (uptr)_dyld_get_shared_cache_range(&cacheLength); + CHECK(cacheStart && cacheLength); + + __block mach_header *dyldHdr = nullptr; + int res = dyld_shared_cache_iterate_text( + uuid, ^(const dyld_shared_cache_dylib_text_info *info) { + CHECK_GE(info->version, 2); + mach_header *hdr = + (mach_header *)(cacheStart + info->textSegmentOffset); + if (IsDyldHdr(hdr)) + dyldHdr = hdr; + }); + CHECK_EQ(res, 0); + + return dyldHdr; +} + const mach_header *get_dyld_hdr() { - if (!dyld_hdr) dyld_hdr = get_dyld_image_header(); + if (!dyld_hdr) { + // On macOS 13+, dyld itself has moved into the shared cache. Looking it up + // via vm_region_recurse_64() causes spins/hangs/crashes. + if (GetMacosAlignedVersion() >= MacosVersion(13, 0)) { + dyld_hdr = GetDyldImageHeaderViaSharedCache(); + if (!dyld_hdr) { + VReport(1, + "Failed to lookup the dyld image header in the shared cache on " + "macOS 13+ (or no shared cache in use). Falling back to " + "lookup via vm_region_recurse_64().\n"); + dyld_hdr = GetDyldImageHeaderViaVMRegion(); + } + } else { + dyld_hdr = GetDyldImageHeaderViaVMRegion(); + } + CHECK(dyld_hdr); + } return dyld_hdr; } @@ -376,4 +442,4 @@ void MemoryMappingLayout::DumpListOfModules( } // namespace __sanitizer -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_solaris.cpp b/libsanitizer/sanitizer_common/sanitizer_procmaps_solaris.cpp index e16c4e938cb..6f43817aedb 100644 --- a/libsanitizer/sanitizer_common/sanitizer_procmaps_solaris.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_solaris.cpp @@ -9,25 +9,32 @@ // Information about the process mappings (Solaris-specific parts). //===----------------------------------------------------------------------===// -// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment. -#undef _FILE_OFFSET_BITS #include "sanitizer_platform.h" #if SANITIZER_SOLARIS -#include "sanitizer_common.h" -#include "sanitizer_procmaps.h" +# include <fcntl.h> +# include <limits.h> +# include <procfs.h> -#include <procfs.h> -#include <limits.h> +# include "sanitizer_common.h" +# include "sanitizer_procmaps.h" namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - if (!ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, - &proc_maps->mmaped_size, &proc_maps->len)) { - proc_maps->data = nullptr; - proc_maps->mmaped_size = 0; - proc_maps->len = 0; - } + uptr fd = internal_open("/proc/self/xmap", O_RDONLY); + CHECK_NE(fd, -1); + uptr Size = internal_filesize(fd); + CHECK_GT(Size, 0); + + // Allow for additional entries by following mmap. + size_t MmapedSize = Size * 4 / 3; + void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()"); + Size = internal_read(fd, VmMap, MmapedSize); + CHECK_NE(Size, -1); + internal_close(fd); + proc_maps->data = (char *)VmMap; + proc_maps->mmaped_size = MmapedSize; + proc_maps->len = Size; } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { @@ -49,21 +56,28 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { segment->protection |= kProtectionWrite; if ((xmapentry->pr_mflags & MA_EXEC) != 0) segment->protection |= kProtectionExecute; + if ((xmapentry->pr_mflags & MA_SHARED) != 0) + segment->protection |= kProtectionShared; if (segment->filename != NULL && segment->filename_size > 0) { char proc_path[PATH_MAX + 1]; - internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s", - xmapentry->pr_mapname); - ssize_t sz = internal_readlink(proc_path, segment->filename, - segment->filename_size - 1); - - // If readlink failed, the map is anonymous. - if (sz == -1) { + // Avoid unnecessary readlink on unnamed entires. + if (xmapentry->pr_mapname[0] == '\0') segment->filename[0] = '\0'; - } else if ((size_t)sz < segment->filename_size) - // readlink doesn't NUL-terminate. - segment->filename[sz] = '\0'; + else { + internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s", + xmapentry->pr_mapname); + ssize_t sz = internal_readlink(proc_path, segment->filename, + segment->filename_size - 1); + + // If readlink failed, the map is anonymous. + if (sz == -1) + segment->filename[0] = '\0'; + else if ((size_t)sz < segment->filename_size) + // readlink doesn't NUL-terminate. + segment->filename[sz] = '\0'; + } } data_.current += sizeof(prxmap_t); diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp index 7386285f34b..d24fae98213 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp @@ -87,8 +87,8 @@ static inline uhwptr *GetCanonicFrame(uptr bp, // Nope, this does not look right either. This means the frame after next does // not have a valid frame pointer, but we can still extract the caller PC. // Unfortunately, there is no way to decide between GCC and LLVM frame - // layouts. Assume GCC. - return bp_prev - 1; + // layouts. Assume LLVM. + return bp_prev; #else return (uhwptr*)bp; #endif @@ -111,24 +111,17 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, IsAligned((uptr)frame, sizeof(*frame)) && size < max_depth) { #ifdef __powerpc__ - // PowerPC ABIs specify that the return address is saved on the - // *caller's* stack frame. Thus we must dereference the back chain - // to find the caller frame before extracting it. + // PowerPC ABIs specify that the return address is saved at offset + // 16 of the *caller's* stack frame. Thus we must dereference the + // back chain to find the caller frame before extracting it. uhwptr *caller_frame = (uhwptr*)frame[0]; if (!IsValidFrame((uptr)caller_frame, stack_top, bottom) || !IsAligned((uptr)caller_frame, sizeof(uhwptr))) break; - // For most ABIs the offset where the return address is saved is two - // register sizes. The exception is the SVR4 ABI, which uses an - // offset of only one register size. -#ifdef _CALL_SYSV - uhwptr pc1 = caller_frame[1]; -#else uhwptr pc1 = caller_frame[2]; -#endif #elif defined(__s390__) uhwptr pc1 = frame[14]; -#elif defined(__riscv) +#elif defined(__loongarch__) || defined(__riscv) // frame[-1] contains the return address uhwptr pc1 = frame[-1]; #else @@ -143,7 +136,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, trace_buffer[size++] = (uptr) pc1; } bottom = (uptr)frame; -#if defined(__riscv) +#if defined(__loongarch__) || defined(__riscv) // frame[-2] contain fp of the previous frame uptr new_bp = (uptr)frame[-2]; #else diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h index 9a5f8fb13a2..ee996c3e07e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h @@ -33,7 +33,7 @@ static const u32 kStackTraceMax = 255; // Fast unwind is the only option on Mac for now; we will need to // revisit this macro when slow unwind works on Mac, see // https://github.com/google/sanitizers/issues/137 -#if SANITIZER_MAC +#if SANITIZER_APPLE # define SANITIZER_CAN_SLOW_UNWIND 0 #else # define SANITIZER_CAN_SLOW_UNWIND 1 diff --git a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cpp index 5ec30803b7a..87f5250db64 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cpp @@ -12,7 +12,7 @@ #include "sanitizer_platform.h" -#if SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__) || \ +#if SANITIZER_APPLE && (defined(__x86_64__) || defined(__aarch64__) || \ defined(__i386)) #include <mach/mach.h> @@ -29,7 +29,7 @@ typedef struct { class SuspendedThreadsListMac final : public SuspendedThreadsList { public: - SuspendedThreadsListMac() : threads_(1024) {} + SuspendedThreadsListMac() = default; tid_t GetThreadID(uptr index) const override; thread_t GetThread(uptr index) const; @@ -176,5 +176,5 @@ PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP( } // namespace __sanitizer -#endif // SANITIZER_MAC && (defined(__x86_64__) || defined(__aarch64__)) || +#endif // SANITIZER_APPLE && (defined(__x86_64__) || defined(__aarch64__)) || // defined(__i386)) diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h index df122ed3425..29a08386d0b 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h @@ -90,9 +90,10 @@ class SymbolizerProcess { // Customizable by subclasses. virtual bool StartSymbolizerSubprocess(); - virtual bool ReadFromSymbolizer(char *buffer, uptr max_length); + virtual bool ReadFromSymbolizer(); // Return the environment to run the symbolizer in. virtual char **GetEnvP() { return GetEnviron(); } + InternalMmapVector<char> &GetBuff() { return buffer_; } private: virtual bool ReachedEndOfOutput(const char *buffer, uptr length) const { @@ -113,8 +114,7 @@ class SymbolizerProcess { fd_t input_fd_; fd_t output_fd_; - static const uptr kBufferSize = 16 * 1024; - char buffer_[kBufferSize]; + InternalMmapVector<char> buffer_; static const uptr kMaxTimesRestarted = 5; static const int kSymbolizerStartupTimeMillis = 10; diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cpp index 8bbd4af0c7c..461fe966136 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cpp @@ -237,7 +237,7 @@ const LoadedModule *Symbolizer::FindModuleForAddress(uptr address) { class LLVMSymbolizerProcess final : public SymbolizerProcess { public: explicit LLVMSymbolizerProcess(const char *path) - : SymbolizerProcess(path, /*use_posix_spawn=*/SANITIZER_MAC) {} + : SymbolizerProcess(path, /*use_posix_spawn=*/SANITIZER_APPLE) {} private: bool ReachedEndOfOutput(const char *buffer, uptr length) const override { @@ -363,14 +363,21 @@ void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res) { } } -// Parses a two-line string in the following format: +// Parses a two- or three-line string in the following format: // <symbol_name> // <start_address> <size> -// Used by LLVMSymbolizer and InternalSymbolizer. +// <filename>:<column> +// Used by LLVMSymbolizer and InternalSymbolizer. LLVMSymbolizer added support +// for symbolizing the third line in D123538, but we support the older two-line +// information as well. void ParseSymbolizeDataOutput(const char *str, DataInfo *info) { str = ExtractToken(str, "\n", &info->name); str = ExtractUptr(str, " ", &info->start); str = ExtractUptr(str, "\n", &info->size); + // Note: If the third line isn't present, these calls will set info.{file, + // line} to empty strings. + str = ExtractToken(str, ":", &info->file); + str = ExtractUptr(str, "\n", &info->line); } static void ParseSymbolizeFrameOutput(const char *str, @@ -500,9 +507,9 @@ const char *SymbolizerProcess::SendCommandImpl(const char *command) { return nullptr; if (!WriteToSymbolizer(command, internal_strlen(command))) return nullptr; - if (!ReadFromSymbolizer(buffer_, kBufferSize)) - return nullptr; - return buffer_; + if (!ReadFromSymbolizer()) + return nullptr; + return buffer_.data(); } bool SymbolizerProcess::Restart() { @@ -513,31 +520,33 @@ bool SymbolizerProcess::Restart() { return StartSymbolizerSubprocess(); } -bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) { - if (max_length == 0) - return true; - uptr read_len = 0; - while (true) { +bool SymbolizerProcess::ReadFromSymbolizer() { + buffer_.clear(); + constexpr uptr max_length = 1024; + bool ret = true; + do { uptr just_read = 0; - bool success = ReadFromFile(input_fd_, buffer + read_len, - max_length - read_len - 1, &just_read); + uptr size_before = buffer_.size(); + buffer_.resize(size_before + max_length); + buffer_.resize(buffer_.capacity()); + bool ret = ReadFromFile(input_fd_, &buffer_[size_before], + buffer_.size() - size_before, &just_read); + + if (!ret) + just_read = 0; + + buffer_.resize(size_before + just_read); + // We can't read 0 bytes, as we don't expect external symbolizer to close // its stdout. - if (!success || just_read == 0) { + if (just_read == 0) { Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_); - return false; - } - read_len += just_read; - if (ReachedEndOfOutput(buffer, read_len)) - break; - if (read_len + 1 == max_length) { - Report("WARNING: Symbolizer buffer too small\n"); - read_len = 0; + ret = false; break; } - } - buffer[read_len] = '\0'; - return true; + } while (!ReachedEndOfOutput(buffer_.data(), buffer_.size())); + buffer_.push_back('\0'); + return ret; } bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) { diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cpp index ac811c8a913..f4f2a036a1e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "sanitizer_allocator_internal.h" #include "sanitizer_mac.h" @@ -202,4 +202,4 @@ bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { } // namespace __sanitizer -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.h index d5abe9d98c1..cea24418290 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.h +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.h @@ -15,7 +15,7 @@ #define SANITIZER_SYMBOLIZER_MAC_H #include "sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "sanitizer_symbolizer_internal.h" @@ -42,6 +42,6 @@ class AtosSymbolizer final : public SymbolizerTool { } // namespace __sanitizer -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE #endif // SANITIZER_SYMBOLIZER_MAC_H diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp index 5f6e4cc3180..b223f6cd01e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp @@ -72,7 +72,6 @@ static swift_demangle_ft swift_demangle_f; // symbolication. static void InitializeSwiftDemangler() { swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle"); - (void)dlerror(); // Cleanup error message in case of failure } // Attempts to demangle a Swift name. The demangler will return nullptr if a @@ -155,7 +154,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() { } if (use_posix_spawn_) { -#if SANITIZER_MAC +#if SANITIZER_APPLE fd_t fd = internal_spawn(argv, const_cast<const char **>(GetEnvP()), &pid); if (fd == kInvalidFd) { Report("WARNING: failed to spawn external symbolizer (errno: %d)\n", @@ -165,9 +164,9 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() { input_fd_ = fd; output_fd_ = fd; -#else // SANITIZER_MAC +#else // SANITIZER_APPLE UNIMPLEMENTED(); -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE } else { fd_t infd[2] = {}, outfd[2] = {}; if (!CreateTwoHighNumberedPipes(infd, outfd)) { @@ -225,24 +224,24 @@ class Addr2LineProcess final : public SymbolizerProcess { bool ReachedEndOfOutput(const char *buffer, uptr length) const override; - bool ReadFromSymbolizer(char *buffer, uptr max_length) override { - if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length)) + bool ReadFromSymbolizer() override { + if (!SymbolizerProcess::ReadFromSymbolizer()) return false; - // The returned buffer is empty when output is valid, but exceeds - // max_length. - if (*buffer == '\0') - return true; + auto &buff = GetBuff(); // We should cut out output_terminator_ at the end of given buffer, // appended by addr2line to mark the end of its meaningful output. // We cannot scan buffer from it's beginning, because it is legal for it // to start with output_terminator_ in case given offset is invalid. So, // scanning from second character. - char *garbage = internal_strstr(buffer + 1, output_terminator_); + char *garbage = internal_strstr(buff.data() + 1, output_terminator_); // This should never be NULL since buffer must end up with // output_terminator_. CHECK(garbage); + // Trim the buffer. - garbage[0] = '\0'; + uintptr_t new_size = garbage - buff.data(); + GetBuff().resize(new_size); + GetBuff().push_back('\0'); return true; } @@ -427,13 +426,13 @@ static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) { VReport(2, "Using llvm-symbolizer at user-specified path: %s\n", path); return new(*allocator) LLVMSymbolizer(path, allocator); } else if (!internal_strcmp(binary_name, "atos")) { -#if SANITIZER_MAC +#if SANITIZER_APPLE VReport(2, "Using atos at user-specified path: %s\n", path); return new(*allocator) AtosSymbolizer(path, allocator); -#else // SANITIZER_MAC +#else // SANITIZER_APPLE Report("ERROR: Using `atos` is only supported on Darwin.\n"); Die(); -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE } else if (!internal_strcmp(binary_name, "addr2line")) { VReport(2, "Using addr2line at user-specified path: %s\n", path); return new(*allocator) Addr2LinePool(path, allocator); @@ -446,12 +445,12 @@ static SymbolizerTool *ChooseExternalSymbolizer(LowLevelAllocator *allocator) { // Otherwise symbolizer program is unknown, let's search $PATH CHECK(path == nullptr); -#if SANITIZER_MAC +#if SANITIZER_APPLE if (const char *found_path = FindPathToBinary("atos")) { VReport(2, "Using atos found at: %s\n", found_path); return new(*allocator) AtosSymbolizer(found_path, allocator); } -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE if (const char *found_path = FindPathToBinary("llvm-symbolizer")) { VReport(2, "Using llvm-symbolizer found at: %s\n", found_path); return new(*allocator) LLVMSymbolizer(found_path, allocator); @@ -488,10 +487,10 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list, list->push_back(tool); } -#if SANITIZER_MAC +#if SANITIZER_APPLE VReport(2, "Using dladdr symbolizer.\n"); list->push_back(new(*allocator) DlAddrSymbolizer()); -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE } Symbolizer *Symbolizer::PlatformInit() { diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp index ac855c8be1c..d5c028e3640 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -94,7 +94,7 @@ void ReportMmapWriteExec(int prot, int flags) { if ((prot & pflags) != pflags) return; -# if SANITIZER_MAC && defined(MAP_JIT) +# if SANITIZER_APPLE && defined(MAP_JIT) if ((flags & MAP_JIT) == MAP_JIT) return; # endif diff --git a/libsanitizer/sanitizer_common/sanitizer_syscall_generic.inc b/libsanitizer/sanitizer_common/sanitizer_syscall_generic.inc index 8829985b5b0..e7f95d33ad0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_syscall_generic.inc +++ b/libsanitizer/sanitizer_common/sanitizer_syscall_generic.inc @@ -13,13 +13,14 @@ // NetBSD uses libc calls directly #if !SANITIZER_NETBSD -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_SOLARIS +#if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_SOLARIS # define SYSCALL(name) SYS_ ## name #else # define SYSCALL(name) __NR_ ## name #endif -#if defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_MAC) +#if (defined(__x86_64__) && (SANITIZER_FREEBSD || SANITIZER_APPLE)) || \ + (defined(__aarch64__) && SANITIZER_FREEBSD) # define internal_syscall __syscall # else # define internal_syscall syscall diff --git a/libsanitizer/sanitizer_common/sanitizer_syscall_linux_loongarch64.inc b/libsanitizer/sanitizer_common/sanitizer_syscall_linux_loongarch64.inc new file mode 100644 index 00000000000..97ca7f2f3f9 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_syscall_linux_loongarch64.inc @@ -0,0 +1,167 @@ +//===-- sanitizer_syscall_linux_loongarch64.inc -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implementations of internal_syscall and internal_iserror for +// Linux/loongarch64. +// +//===----------------------------------------------------------------------===// + +// About local register variables: +// https://gcc.gnu.org/onlinedocs/gcc/Local-Register-Variables.html#Local-Register-Variables +// +// Kernel ABI... +// syscall number is passed in a7 +// (http://man7.org/linux/man-pages/man2/syscall.2.html) results are return in +// a0 and a1 (http://man7.org/linux/man-pages/man2/syscall.2.html) arguments +// are passed in: a0-a7 (confirmed by inspecting glibc sources). +#define SYSCALL(name) __NR_##name + +#define INTERNAL_SYSCALL_CLOBBERS "memory" + +static uptr __internal_syscall(u64 nr) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0"); + __asm__ volatile("syscall 0\n\t" + : "=r"(a0) + : "r"(a7) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall0(n) (__internal_syscall)(n) + +static uptr __internal_syscall(u64 nr, u64 arg1) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall1(n, a1) (__internal_syscall)(n, (u64)(a1)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall2(n, a1, a2) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall3(n, a1, a2, a3) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, + u64 arg4) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + register u64 a3 asm("a3") = arg4; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2), "r"(a3) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall4(n, a1, a2, a3, a4) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4, + long arg5) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + register u64 a3 asm("a3") = arg4; + register u64 a4 asm("a4") = arg5; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall5(n, a1, a2, a3, a4, a5) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u64)(a5)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4, + long arg5, long arg6) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + register u64 a3 asm("a3") = arg4; + register u64 a4 asm("a4") = arg5; + register u64 a5 asm("a5") = arg6; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u64)(a5), (long)(a6)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4, + long arg5, long arg6, long arg7) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + register u64 a3 asm("a3") = arg4; + register u64 a4 asm("a4") = arg5; + register u64 a5 asm("a5") = arg6; + register u64 a6 asm("a6") = arg7; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5), + "r"(a6) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall7(n, a1, a2, a3, a4, a5, a6, a7) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u64)(a5), (long)(a6), (long)(a7)) + +#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n +#define __SYSCALL_NARGS(...) \ + __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) +#define __SYSCALL_CONCAT_X(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b) +#define __SYSCALL_DISP(b, ...) \ + __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) + +// Helper function used to avoid clobbering of errno. +bool internal_iserror(uptr retval, int *internal_errno) { + if (retval >= (uptr)-4095) { + if (internal_errno) + *internal_errno = -retval; + return true; + } + return false; +} diff --git a/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp index b2628dcc4dc..72f025a7d30 100644 --- a/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp @@ -58,7 +58,7 @@ unwind_backtrace_signal_arch_func unwind_backtrace_signal_arch; #endif uptr Unwind_GetIP(struct _Unwind_Context *ctx) { -#if defined(__arm__) && !SANITIZER_MAC +#if defined(__arm__) && !SANITIZER_APPLE uptr val; _Unwind_VRS_Result res = _Unwind_VRS_Get(ctx, _UVRSC_CORE, 15 /* r15 = PC */, _UVRSD_UINT32, &val); diff --git a/libsanitizer/sanitizer_common/sanitizer_vector.h b/libsanitizer/sanitizer_common/sanitizer_vector.h index 31216f3ec3a..79ff275660d 100644 --- a/libsanitizer/sanitizer_common/sanitizer_vector.h +++ b/libsanitizer/sanitizer_common/sanitizer_vector.h @@ -83,8 +83,8 @@ class Vector { } EnsureSize(size); if (old_size < size) { - for (uptr i = old_size; i < size; i++) - internal_memset(&begin_[i], 0, sizeof(begin_[i])); + internal_memset(&begin_[old_size], 0, + sizeof(begin_[old_size]) * (size - old_size)); } } diff --git a/libsanitizer/sanitizer_common/sanitizer_win.cpp b/libsanitizer/sanitizer_common/sanitizer_win.cpp index 53770331199..e0568c9b62d 100644 --- a/libsanitizer/sanitizer_common/sanitizer_win.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_win.cpp @@ -131,6 +131,11 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, } #endif // #if !SANITIZER_GO +bool ErrorIsOOM(error_t err) { + // TODO: This should check which `err`s correspond to OOM. + return false; +} + void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); if (rv == 0) @@ -229,6 +234,17 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, return (void *)mapped_addr; } +// ZeroMmapFixedRegion zero's out a region of memory previously returned from a +// call to one of the MmapFixed* helpers. On non-windows systems this would be +// done with another mmap, but on windows remapping is not an option. +// VirtualFree(DECOMMIT)+VirtualAlloc(RECOMMIT) would also be a way to zero the +// memory, but we can't do this atomically, so instead we fall back to using +// internal_memset. +bool ZeroMmapFixedRegion(uptr fixed_addr, uptr size) { + internal_memset((void*) fixed_addr, 0, size); + return true; +} + bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) { // FIXME: is this really "NoReserve"? On Win32 this does not matter much, // but on Win64 it does. @@ -1089,10 +1105,6 @@ void InitializePlatformEarly() { // Do nothing. } -void MaybeReexec() { - // No need to re-exec on Windows. -} - void CheckASLR() { // Do nothing } diff --git a/libsanitizer/tsan/tsan_dense_alloc.h b/libsanitizer/tsan/tsan_dense_alloc.h index 7a39a39d51d..2eaff39057b 100644 --- a/libsanitizer/tsan/tsan_dense_alloc.h +++ b/libsanitizer/tsan/tsan_dense_alloc.h @@ -85,14 +85,7 @@ class DenseSlabAlloc { } void FlushCache(Cache *c) { - if (!c->pos) - return; - SpinMutexLock lock(&mtx_); - while (c->pos) { - IndexT idx = c->cache[--c->pos]; - *(IndexT*)Map(idx) = freelist_; - freelist_ = idx; - } + while (c->pos) Drain(c); } void InitCache(Cache *c) { @@ -106,7 +99,7 @@ class DenseSlabAlloc { template <typename Func> void ForEach(Func func) { - SpinMutexLock lock(&mtx_); + Lock lock(&mtx_); uptr fillpos = atomic_load_relaxed(&fillpos_); for (uptr l1 = 0; l1 < fillpos; l1++) { for (IndexT l2 = l1 == 0 ? 1 : 0; l2 < kL2Size; l2++) func(&map_[l1][l2]); @@ -115,48 +108,86 @@ class DenseSlabAlloc { private: T *map_[kL1Size]; - SpinMutex mtx_; - IndexT freelist_ = {0}; + Mutex mtx_; + // The freelist is organized as a lock-free stack of batches of nodes. + // The stack itself uses Block::next links, while the batch within each + // stack node uses Block::batch links. + // Low 32-bits of freelist_ is the node index, top 32-bits is ABA-counter. + atomic_uint64_t freelist_ = {0}; atomic_uintptr_t fillpos_ = {0}; const char *const name_; - void Refill(Cache *c) { - SpinMutexLock lock(&mtx_); - if (freelist_ == 0) { - uptr fillpos = atomic_load_relaxed(&fillpos_); - if (fillpos == kL1Size) { - Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", - name_, kL1Size, kL2Size); - Die(); - } - VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_, - fillpos, kL1Size, kL2Size); - T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_); - // Reserve 0 as invalid index. - IndexT start = fillpos == 0 ? 1 : 0; - for (IndexT i = start; i < kL2Size; i++) { - new(batch + i) T; - *(IndexT *)(batch + i) = i + 1 + fillpos * kL2Size; - } - *(IndexT*)(batch + kL2Size - 1) = 0; - freelist_ = fillpos * kL2Size + start; - map_[fillpos] = batch; - atomic_store_relaxed(&fillpos_, fillpos + 1); - } - for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) { - IndexT idx = freelist_; + struct Block { + IndexT next; + IndexT batch; + }; + + Block *MapBlock(IndexT idx) { return reinterpret_cast<Block *>(Map(idx)); } + + static constexpr u64 kCounterInc = 1ull << 32; + static constexpr u64 kCounterMask = ~(kCounterInc - 1); + + NOINLINE void Refill(Cache *c) { + // Pop 1 batch of nodes from the freelist. + IndexT idx; + u64 xchg; + u64 cmp = atomic_load(&freelist_, memory_order_acquire); + do { + idx = static_cast<IndexT>(cmp); + if (!idx) + return AllocSuperBlock(c); + Block *ptr = MapBlock(idx); + xchg = ptr->next | (cmp & kCounterMask); + } while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg, + memory_order_acq_rel)); + // Unpack it into c->cache. + while (idx) { c->cache[c->pos++] = idx; - freelist_ = *(IndexT*)Map(idx); + idx = MapBlock(idx)->batch; } } - void Drain(Cache *c) { - SpinMutexLock lock(&mtx_); - for (uptr i = 0; i < Cache::kSize / 2; i++) { + NOINLINE void Drain(Cache *c) { + // Build a batch of at most Cache::kSize / 2 nodes linked by Block::batch. + IndexT head_idx = 0; + for (uptr i = 0; i < Cache::kSize / 2 && c->pos; i++) { IndexT idx = c->cache[--c->pos]; - *(IndexT*)Map(idx) = freelist_; - freelist_ = idx; + Block *ptr = MapBlock(idx); + ptr->batch = head_idx; + head_idx = idx; + } + // Push it onto the freelist stack. + Block *head = MapBlock(head_idx); + u64 xchg; + u64 cmp = atomic_load(&freelist_, memory_order_acquire); + do { + head->next = static_cast<IndexT>(cmp); + xchg = head_idx | (cmp & kCounterMask) + kCounterInc; + } while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg, + memory_order_acq_rel)); + } + + NOINLINE void AllocSuperBlock(Cache *c) { + Lock lock(&mtx_); + uptr fillpos = atomic_load_relaxed(&fillpos_); + if (fillpos == kL1Size) { + Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", name_, kL1Size, + kL2Size); + Die(); + } + VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_, + fillpos, kL1Size, kL2Size); + T *batch = (T *)MmapOrDie(kL2Size * sizeof(T), name_); + map_[fillpos] = batch; + // Reserve 0 as invalid index. + for (IndexT i = fillpos ? 0 : 1; i < kL2Size; i++) { + new (batch + i) T; + c->cache[c->pos++] = i + fillpos * kL2Size; + if (c->pos == Cache::kSize) + Drain(c); } + atomic_store_relaxed(&fillpos_, fillpos + 1); + CHECK(c->pos); } }; diff --git a/libsanitizer/tsan/tsan_dispatch_defs.h b/libsanitizer/tsan/tsan_dispatch_defs.h index 94e0b50fed3..54c0b0ba4b4 100644 --- a/libsanitizer/tsan/tsan_dispatch_defs.h +++ b/libsanitizer/tsan/tsan_dispatch_defs.h @@ -56,7 +56,7 @@ extern const dispatch_block_t _dispatch_data_destructor_munmap; # define DISPATCH_NOESCAPE #endif -#if SANITIZER_MAC +#if SANITIZER_APPLE # define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import)) #else # define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak)) diff --git a/libsanitizer/tsan/tsan_fd.cpp b/libsanitizer/tsan/tsan_fd.cpp index cf8f491fdbf..ab295a69dce 100644 --- a/libsanitizer/tsan/tsan_fd.cpp +++ b/libsanitizer/tsan/tsan_fd.cpp @@ -34,6 +34,7 @@ struct FdDesc { atomic_uintptr_t aux_sync; // FdSync* Tid creation_tid; StackID creation_stack; + bool closed; }; struct FdContext { @@ -120,6 +121,7 @@ static void init(ThreadState *thr, uptr pc, int fd, FdSync *s, } d->creation_tid = thr->tid; d->creation_stack = CurrentStackId(thr, pc); + d->closed = false; // This prevents false positives on fd_close_norace3.cpp test. // The mechanics of the false positive are not completely clear, // but it happens only if global reset is enabled (flush_memory_ms=1) @@ -155,7 +157,7 @@ void FdOnFork(ThreadState *thr, uptr pc) { } } -bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack) { +bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack, bool *closed) { for (int l1 = 0; l1 < kTableSizeL1; l1++) { FdDesc *tab = (FdDesc*)atomic_load(&fdctx.tab[l1], memory_order_relaxed); if (tab == 0) @@ -166,6 +168,7 @@ bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack) { *fd = l1 * kTableSizeL1 + l2; *tid = d->creation_tid; *stack = d->creation_stack; + *closed = d->closed; return true; } } @@ -242,8 +245,9 @@ void FdClose(ThreadState *thr, uptr pc, int fd, bool write) { reinterpret_cast<FdSync *>( atomic_load(&d->aux_sync, memory_order_relaxed))); atomic_store(&d->aux_sync, 0, memory_order_relaxed); - d->creation_tid = kInvalidTid; - d->creation_stack = kInvalidStackID; + d->closed = true; + d->creation_tid = thr->tid; + d->creation_stack = CurrentStackId(thr, pc); } void FdFileCreate(ThreadState *thr, uptr pc, int fd) { diff --git a/libsanitizer/tsan/tsan_fd.h b/libsanitizer/tsan/tsan_fd.h index 92625dc4b4a..dddc1d2ab24 100644 --- a/libsanitizer/tsan/tsan_fd.h +++ b/libsanitizer/tsan/tsan_fd.h @@ -54,7 +54,7 @@ void FdSocketCreate(ThreadState *thr, uptr pc, int fd); void FdSocketAccept(ThreadState *thr, uptr pc, int fd, int newfd); void FdSocketConnecting(ThreadState *thr, uptr pc, int fd); void FdSocketConnect(ThreadState *thr, uptr pc, int fd); -bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack); +bool FdLocation(uptr addr, int *fd, Tid *tid, StackID *stack, bool *closed); void FdOnFork(ThreadState *thr, uptr pc); uptr File2addr(const char *path); diff --git a/libsanitizer/tsan/tsan_flags.inc b/libsanitizer/tsan/tsan_flags.inc index 32cf3bbf152..731d776cc89 100644 --- a/libsanitizer/tsan/tsan_flags.inc +++ b/libsanitizer/tsan/tsan_flags.inc @@ -23,10 +23,6 @@ TSAN_FLAG(bool, enable_annotations, true, TSAN_FLAG(bool, suppress_equal_stacks, true, "Suppress a race report if we've already output another race report " "with the same stack.") -TSAN_FLAG(bool, suppress_equal_addresses, true, - "Suppress a race report if we've already output another race report " - "on the same address.") - TSAN_FLAG(bool, report_bugs, true, "Turns off bug reporting entirely (useful for benchmarking).") TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?") @@ -74,9 +70,9 @@ TSAN_FLAG(int, io_sync, 1, TSAN_FLAG(bool, die_after_fork, true, "Die after multi-threaded fork if the child creates new threads.") TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") -TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_MAC ? true : false, +TSAN_FLAG(bool, ignore_interceptors_accesses, SANITIZER_APPLE ? true : false, "Ignore reads and writes from all interceptors.") -TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_MAC ? true : false, +TSAN_FLAG(bool, ignore_noninstrumented_modules, SANITIZER_APPLE ? true : false, "Interceptors should only detect races when called from instrumented " "modules.") TSAN_FLAG(bool, shared_ptr_interceptor, true, diff --git a/libsanitizer/tsan/tsan_interceptors_libdispatch.cpp b/libsanitizer/tsan/tsan_interceptors_libdispatch.cpp index cbbb7ecb239..88d5f0a4811 100644 --- a/libsanitizer/tsan/tsan_interceptors_libdispatch.cpp +++ b/libsanitizer/tsan/tsan_interceptors_libdispatch.cpp @@ -19,7 +19,7 @@ #include "BlocksRuntime/Block.h" #include "tsan_dispatch_defs.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE # include <Availability.h> #endif @@ -225,7 +225,7 @@ DISPATCH_INTERCEPT(dispatch_barrier, true) // dispatch_async_and_wait() and friends were introduced in macOS 10.14. // Linking of these interceptors fails when using an older SDK. -#if !SANITIZER_MAC || defined(__MAC_10_14) +#if !SANITIZER_APPLE || defined(__MAC_10_14) // macOS 10.14 is greater than our minimal deployment target. To ensure we // generate a weak reference so the TSan dylib continues to work on older // systems, we need to forward declare the intercepted functions as "weak diff --git a/libsanitizer/tsan/tsan_interceptors_mac.cpp b/libsanitizer/tsan/tsan_interceptors_mac.cpp index ed064150d00..1ee47bcd123 100644 --- a/libsanitizer/tsan/tsan_interceptors_mac.cpp +++ b/libsanitizer/tsan/tsan_interceptors_mac.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "interception/interception.h" #include "tsan_interceptors.h" @@ -518,4 +518,4 @@ STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag, } // namespace __tsan -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/tsan/tsan_interceptors_posix.cpp b/libsanitizer/tsan/tsan_interceptors_posix.cpp index 60ca9633868..17f6b1f472d 100644 --- a/libsanitizer/tsan/tsan_interceptors_posix.cpp +++ b/libsanitizer/tsan/tsan_interceptors_posix.cpp @@ -35,7 +35,7 @@ using namespace __tsan; -#if SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_FREEBSD || SANITIZER_APPLE #define stdout __stdoutp #define stderr __stderrp #endif @@ -102,14 +102,14 @@ extern __sanitizer_FILE __sF[]; #else extern __sanitizer_FILE *stdout, *stderr; #endif -#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD +#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; #else const int PTHREAD_MUTEX_RECURSIVE = 2; const int PTHREAD_MUTEX_RECURSIVE_NP = 2; #endif -#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD +#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD const int EPOLL_CTL_ADD = 1; #endif const int SIGILL = 4; @@ -119,7 +119,7 @@ const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; -#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD +#if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SIGBUS = 10; const int SIGSYS = 12; #else @@ -129,7 +129,7 @@ const int SIGSYS = 31; void *const MAP_FAILED = (void*)-1; #if SANITIZER_NETBSD const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; -#elif !SANITIZER_MAC +#elif !SANITIZER_APPLE const int PTHREAD_BARRIER_SERIAL_THREAD = -1; #endif const int MAP_FIXED = 0x10; @@ -142,7 +142,7 @@ typedef __sanitizer::u16 mode_t; # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ # define F_TEST 3 /* Test a region for other processes locks. */ -#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD +#if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SA_SIGINFO = 0x40; const int SIG_SETMASK = 3; #elif defined(__mips__) @@ -189,7 +189,7 @@ struct InterceptorContext { // in a single cache line if possible (it's accessed in every interceptor). ALIGNED(64) LibIgnore libignore; __sanitizer_sigaction sigactions[kSigCount]; -#if !SANITIZER_MAC && !SANITIZER_NETBSD +#if !SANITIZER_APPLE && !SANITIZER_NETBSD unsigned finalize_key; #endif @@ -461,7 +461,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), return res; } -#if !SANITIZER_MAC && !SANITIZER_NETBSD +#if !SANITIZER_APPLE && !SANITIZER_NETBSD static void on_exit_callback_installed_at(int status, void *arg) { ThreadState *thr = cur_thread(); AtExitCtx *ctx = (AtExitCtx*)arg; @@ -553,11 +553,11 @@ static void LongJmp(ThreadState *thr, uptr *env) { // FIXME: put everything below into a common extern "C" block? extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); } -#if SANITIZER_MAC +#if SANITIZER_APPLE TSAN_INTERCEPTOR(int, setjmp, void *env); TSAN_INTERCEPTOR(int, _setjmp, void *env); TSAN_INTERCEPTOR(int, sigsetjmp, void *env); -#else // SANITIZER_MAC +#else // SANITIZER_APPLE #if SANITIZER_NETBSD #define setjmp_symname __setjmp14 @@ -619,7 +619,7 @@ DEFINE_REAL(int, sigsetjmp_symname, void *env) #if !SANITIZER_NETBSD DEFINE_REAL(int, __sigsetjmp, void *env) #endif -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE #if SANITIZER_NETBSD #define longjmp_symname __longjmp14 @@ -658,7 +658,7 @@ TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { } #endif -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(void*, malloc, uptr size) { if (in_symbolizer()) return InternalAlloc(size); @@ -816,7 +816,7 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { #define TSAN_MAYBE_INTERCEPT_MEMALIGN #endif -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { if (in_symbolizer()) return InternalAlloc(sz, nullptr, align); @@ -847,7 +847,7 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { #define TSAN_MAYBE_INTERCEPT_PVALLOC #endif -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { if (in_symbolizer()) { void *p = InternalAlloc(sz, nullptr, align); @@ -919,7 +919,7 @@ static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g, // these interceptors with INTERFACE_ATTRIBUTE. // On OS X, we don't support statically linking, so we just use a regular // interceptor. -#if SANITIZER_MAC +#if SANITIZER_APPLE #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR #else #define STDCXX_INTERCEPTOR(rettype, name, ...) \ @@ -962,7 +962,7 @@ void PlatformCleanUpThreadState(ThreadState *thr) { } } // namespace __tsan -#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD +#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD static void thread_finalize(void *v) { uptr iter = (uptr)v; if (iter > 1) { @@ -994,7 +994,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { ThreadState *thr = cur_thread_init(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; -#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD +#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD ThreadIgnoreBegin(thr, 0); if (pthread_setspecific(interceptor_ctx()->finalize_key, (void *)GetPthreadDestructorIterations())) { @@ -1102,7 +1102,7 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { { SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); -#if !SANITIZER_MAC && !SANITIZER_ANDROID +#if !SANITIZER_APPLE && !SANITIZER_ANDROID CHECK_EQ(thr, &cur_thread_placeholder); #endif } @@ -1271,7 +1271,7 @@ INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m, #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT #endif -#if SANITIZER_MAC +#if SANITIZER_APPLE INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m, void *reltime) { void *cond = init_cond(c); @@ -1348,7 +1348,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { return res; } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); int res = REAL(pthread_mutex_timedlock)(m, abstime); @@ -1359,7 +1359,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { } #endif -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); int res = REAL(pthread_spin_init)(m, pshared); @@ -1442,7 +1442,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { return res; } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); @@ -1472,7 +1472,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { return res; } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); @@ -1490,7 +1490,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { return res; } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); @@ -1524,7 +1524,7 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { return errno_EINVAL; atomic_uint32_t *a; - if (SANITIZER_MAC) + if (SANITIZER_APPLE) a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t))); else if (SANITIZER_NETBSD) a = static_cast<atomic_uint32_t*> @@ -1534,7 +1534,7 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks // result in crashes due to too little stack space. - if (guard_acquire(thr, pc, a, !SANITIZER_MAC)) { + if (guard_acquire(thr, pc, a, !SANITIZER_APPLE)) { (*f)(); guard_release(thr, pc, a, kGuardDone); } @@ -1661,7 +1661,7 @@ TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { return newfd2; } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); int newfd2 = REAL(dup3)(oldfd, newfd, flags); @@ -1805,7 +1805,7 @@ TSAN_INTERCEPTOR(int, pipe, int *pipefd) { return res; } -#if !SANITIZER_MAC +#if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); int res = REAL(pipe2)(pipefd, flags); @@ -2263,7 +2263,7 @@ TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags, } #endif -#if !SANITIZER_MAC && !SANITIZER_ANDROID +#if !SANITIZER_APPLE && !SANITIZER_ANDROID typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data); struct dl_iterate_phdr_data { @@ -2320,7 +2320,7 @@ struct TsanInterceptorContext { const uptr pc; }; -#if !SANITIZER_MAC +#if !SANITIZER_APPLE static void HandleRecvmsg(ThreadState *thr, uptr pc, __sanitizer_msghdr *msg) { int fds[64]; @@ -2460,7 +2460,7 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, off); \ } while (false) -#if !SANITIZER_MAC +#if !SANITIZER_APPLE #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, msg) @@ -2521,7 +2521,7 @@ int sigaction_impl(int sig, const __sanitizer_sigaction *act, sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags; internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, sizeof(sigactions[sig].sa_mask)); -#if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD +#if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD sigactions[sig].sa_restorer = act->sa_restorer; #endif internal_memcpy(&newact, act, sizeof(newact)); @@ -2568,7 +2568,7 @@ struct ScopedSyscall { } }; -#if !SANITIZER_FREEBSD && !SANITIZER_MAC +#if !SANITIZER_FREEBSD && !SANITIZER_APPLE static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { TSAN_SYSCALL(); MemoryAccessRange(thr, pc, p, s, write); @@ -2772,7 +2772,7 @@ static void finalize(void *arg) { Die(); } -#if !SANITIZER_MAC && !SANITIZER_ANDROID +#if !SANITIZER_APPLE && !SANITIZER_ANDROID static void unreachable() { Report("FATAL: ThreadSanitizer: unreachable called\n"); Die(); @@ -2783,7 +2783,7 @@ static void unreachable() { SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} void InitializeInterceptors() { -#if !SANITIZER_MAC +#if !SANITIZER_APPLE // We need to setup it early, because functions like dlsym() can call it. REAL(memset) = internal_memset; REAL(memcpy) = internal_memcpy; @@ -2795,7 +2795,7 @@ void InitializeInterceptors() { InitializeSignalInterceptors(); InitializeLibdispatchInterceptors(); -#if !SANITIZER_MAC +#if !SANITIZER_APPLE // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. using __interception::InterceptFunction; @@ -2948,7 +2948,7 @@ void InitializeInterceptors() { TSAN_MAYBE_INTERCEPT__LWP_EXIT; TSAN_MAYBE_INTERCEPT_THR_EXIT; -#if !SANITIZER_MAC && !SANITIZER_ANDROID +#if !SANITIZER_APPLE && !SANITIZER_ANDROID // Need to setup it, because interceptors check that the function is resolved. // But atexit is emitted directly into the module, so can't be resolved. REAL(atexit) = (int(*)(void(*)()))unreachable; @@ -2963,7 +2963,7 @@ void InitializeInterceptors() { Die(); } -#if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD +#if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) { Printf("ThreadSanitizer: failed to create thread key\n"); Die(); diff --git a/libsanitizer/tsan/tsan_malloc_mac.cpp b/libsanitizer/tsan/tsan_malloc_mac.cpp index 0e861bf1f96..ac844ae8a44 100644 --- a/libsanitizer/tsan/tsan_malloc_mac.cpp +++ b/libsanitizer/tsan/tsan_malloc_mac.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "sanitizer_common/sanitizer_errno.h" #include "tsan_interceptors.h" diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h index 233bf0a39df..7c13c733513 100644 --- a/libsanitizer/tsan/tsan_platform.h +++ b/libsanitizer/tsan/tsan_platform.h @@ -394,6 +394,7 @@ struct MappingGo48 { 0300 0000 0000 - 0700 0000 0000: - 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects) 07d0 0000 0000 - 8000 0000 0000: - +PIE binaries currently not supported, but it should be theoretically possible. */ struct MappingGoWindows { @@ -587,7 +588,7 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) { #else // SANITIZER_GO # if SANITIZER_IOS && !SANITIZER_IOSSIM return Func::template Apply<MappingAppleAarch64>(arg); -# elif defined(__x86_64__) || SANITIZER_MAC +# elif defined(__x86_64__) || SANITIZER_APPLE return Func::template Apply<Mapping48AddressSpace>(arg); # elif defined(__aarch64__) switch (vmaSize) { diff --git a/libsanitizer/tsan/tsan_platform_linux.cpp b/libsanitizer/tsan/tsan_platform_linux.cpp index 17dbdff8a53..807f6be2eee 100644 --- a/libsanitizer/tsan/tsan_platform_linux.cpp +++ b/libsanitizer/tsan/tsan_platform_linux.cpp @@ -402,7 +402,11 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) { #elif defined(__powerpc__) # define LONG_JMP_SP_ENV_SLOT 0 #elif SANITIZER_FREEBSD -# define LONG_JMP_SP_ENV_SLOT 2 +# ifdef __aarch64__ +# define LONG_JMP_SP_ENV_SLOT 1 +# else +# define LONG_JMP_SP_ENV_SLOT 2 +# endif #elif SANITIZER_LINUX # ifdef __aarch64__ # define LONG_JMP_SP_ENV_SLOT 13 diff --git a/libsanitizer/tsan/tsan_platform_mac.cpp b/libsanitizer/tsan/tsan_platform_mac.cpp index 44b98d46cfb..1aac0fb2752 100644 --- a/libsanitizer/tsan/tsan_platform_mac.cpp +++ b/libsanitizer/tsan/tsan_platform_mac.cpp @@ -12,7 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_platform.h" -#if SANITIZER_MAC +#if SANITIZER_APPLE #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" @@ -200,44 +200,26 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { # if !SANITIZER_GO void InitializeShadowMemoryPlatform() { } -// On OS X, GCD worker threads are created without a call to pthread_create. We -// need to properly register these threads with ThreadCreate and ThreadStart. -// These threads don't have a parent thread, as they are created "spuriously". -// We're using a libpthread API that notifies us about a newly created thread. -// The `thread == pthread_self()` check indicates this is actually a worker -// thread. If it's just a regular thread, this hook is called on the parent -// thread. -typedef void (*pthread_introspection_hook_t)(unsigned int event, - pthread_t thread, void *addr, - size_t size); -extern "C" pthread_introspection_hook_t pthread_introspection_hook_install( - pthread_introspection_hook_t hook); -static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1; -static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3; -static pthread_introspection_hook_t prev_pthread_introspection_hook; -static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, - void *addr, size_t size) { - if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) { - if (thread == pthread_self()) { - // The current thread is a newly created GCD worker thread. - ThreadState *thr = cur_thread(); - Processor *proc = ProcCreate(); - ProcWire(proc, thr); - ThreadState *parent_thread_state = nullptr; // No parent. - Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); - CHECK_NE(tid, kMainTid); - ThreadStart(thr, tid, GetTid(), ThreadType::Worker); - } - } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { - CHECK_EQ(thread, pthread_self()); +// Register GCD worker threads, which are created without an observable call to +// pthread_create(). +static void ThreadCreateCallback(uptr thread, bool gcd_worker) { + if (gcd_worker) { ThreadState *thr = cur_thread(); - if (thr->tctx) { - DestroyThreadState(); - } + Processor *proc = ProcCreate(); + ProcWire(proc, thr); + ThreadState *parent_thread_state = nullptr; // No parent. + Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); + CHECK_NE(tid, kMainTid); + ThreadStart(thr, tid, GetTid(), ThreadType::Worker); } +} - if (prev_pthread_introspection_hook != nullptr) - prev_pthread_introspection_hook(event, thread, addr, size); +// Destroy thread state for *all* threads. +static void ThreadTerminateCallback(uptr thread) { + ThreadState *thr = cur_thread(); + if (thr->tctx) { + DestroyThreadState(); + } } #endif @@ -261,8 +243,11 @@ void InitializePlatform() { InitializeThreadStateStorage(); - prev_pthread_introspection_hook = - pthread_introspection_hook_install(&my_pthread_introspection_hook); + ThreadEventCallbacks callbacks = { + .create = ThreadCreateCallback, + .terminate = ThreadTerminateCallback, + }; + InstallPthreadIntrospectionHook(callbacks); #endif if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) { @@ -316,4 +301,4 @@ int call_pthread_cancel_with_cleanup(int (*fn)(void *arg), } // namespace __tsan -#endif // SANITIZER_MAC +#endif // SANITIZER_APPLE diff --git a/libsanitizer/tsan/tsan_report.cpp b/libsanitizer/tsan/tsan_report.cpp index 9f151279b60..9b03adc16b9 100644 --- a/libsanitizer/tsan/tsan_report.cpp +++ b/libsanitizer/tsan/tsan_report.cpp @@ -98,7 +98,7 @@ static const char *ReportTypeString(ReportType typ, uptr tag) { UNREACHABLE("missing case"); } -#if SANITIZER_MAC +#if SANITIZER_APPLE static const char *const kInterposedFunctionPrefix = "wrap_"; #else static const char *const kInterposedFunctionPrefix = "__interceptor_"; @@ -200,8 +200,9 @@ static void PrintLocation(const ReportLocation *loc) { } else if (loc->type == ReportLocationTLS) { Printf(" Location is TLS of %s.\n\n", thread_name(thrbuf, loc->tid)); } else if (loc->type == ReportLocationFD) { - Printf(" Location is file descriptor %d created by %s at:\n", - loc->fd, thread_name(thrbuf, loc->tid)); + Printf(" Location is file descriptor %d %s by %s at:\n", loc->fd, + loc->fd_closed ? "destroyed" : "created", + thread_name(thrbuf, loc->tid)); print_stack = true; } Printf("%s", d.Default()); diff --git a/libsanitizer/tsan/tsan_report.h b/libsanitizer/tsan/tsan_report.h index 718eacde7ec..3c88864af14 100644 --- a/libsanitizer/tsan/tsan_report.h +++ b/libsanitizer/tsan/tsan_report.h @@ -76,6 +76,7 @@ struct ReportLocation { uptr external_tag = 0; Tid tid = kInvalidTid; int fd = 0; + bool fd_closed = false; bool suppressable = false; ReportStack *stack = nullptr; }; diff --git a/libsanitizer/tsan/tsan_rtl.cpp b/libsanitizer/tsan/tsan_rtl.cpp index 1d6fc725266..db3d94518b8 100644 --- a/libsanitizer/tsan/tsan_rtl.cpp +++ b/libsanitizer/tsan/tsan_rtl.cpp @@ -45,7 +45,7 @@ void (*on_initialize)(void); int (*on_finalize)(int); #endif -#if !SANITIZER_GO && !SANITIZER_MAC +#if !SANITIZER_GO && !SANITIZER_APPLE __attribute__((tls_model("initial-exec"))) THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED( SANITIZER_CACHE_LINE_SIZE); @@ -197,30 +197,45 @@ static void DoResetImpl(uptr epoch) { } DPrintf("Resetting shadow...\n"); - if (!MmapFixedSuperNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), - "shadow")) { + auto shadow_begin = ShadowBeg(); + auto shadow_end = ShadowEnd(); +#if SANITIZER_GO + CHECK_NE(0, ctx->mapped_shadow_begin); + shadow_begin = ctx->mapped_shadow_begin; + shadow_end = ctx->mapped_shadow_end; + VPrintf(2, "shadow_begin-shadow_end: (0x%zx-0x%zx)\n", + shadow_begin, shadow_end); +#endif + +#if SANITIZER_WINDOWS + auto resetFailed = + !ZeroMmapFixedRegion(shadow_begin, shadow_end - shadow_begin); +#else + auto resetFailed = + !MmapFixedSuperNoReserve(shadow_begin, shadow_end-shadow_begin, "shadow"); +#endif + if (resetFailed) { Printf("failed to reset shadow memory\n"); Die(); } DPrintf("Resetting meta shadow...\n"); ctx->metamap.ResetClocks(); + StoreShadow(&ctx->last_spurious_race, Shadow::kEmpty); ctx->resetting = false; } // Clang does not understand locking all slots in the loop: // error: expecting mutex 'slot.mtx' to be held at start of each loop void DoReset(ThreadState* thr, uptr epoch) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { - { - for (auto& slot : ctx->slots) { - slot.mtx.Lock(); - if (UNLIKELY(epoch == 0)) - epoch = ctx->global_epoch; - if (UNLIKELY(epoch != ctx->global_epoch)) { - // Epoch can't change once we've locked the first slot. - CHECK_EQ(slot.sid, 0); - slot.mtx.Unlock(); - return; - } + for (auto& slot : ctx->slots) { + slot.mtx.Lock(); + if (UNLIKELY(epoch == 0)) + epoch = ctx->global_epoch; + if (UNLIKELY(epoch != ctx->global_epoch)) { + // Epoch can't change once we've locked the first slot. + CHECK_EQ(slot.sid, 0); + slot.mtx.Unlock(); + return; } } DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch); @@ -370,7 +385,6 @@ Context::Context() }), racy_mtx(MutexTypeRacy), racy_stacks(), - racy_addresses(), fired_suppressions_mtx(MutexTypeFired), slot_mtx(MutexTypeSlots), resetting() { @@ -559,18 +573,50 @@ void UnmapShadow(ThreadState *thr, uptr addr, uptr size) { #endif void MapShadow(uptr addr, uptr size) { + // Ensure thead registry lock held, so as to synchronize + // with DoReset, which also access the mapped_shadow_* ctxt fields. + ThreadRegistryLock lock0(&ctx->thread_registry); + static bool data_mapped = false; + +#if !SANITIZER_GO // Global data is not 64K aligned, but there are no adjacent mappings, // so we can get away with unaligned mapping. // CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment const uptr kPageSize = GetPageSizeCached(); uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), kPageSize); uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), kPageSize); - if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, - "shadow")) + if (!MmapFixedNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow")) Die(); +#else + uptr shadow_begin = RoundDownTo((uptr)MemToShadow(addr), (64 << 10)); + uptr shadow_end = RoundUpTo((uptr)MemToShadow(addr + size), (64 << 10)); + VPrintf(2, "MapShadow for (0x%zx-0x%zx), begin/end: (0x%zx-0x%zx)\n", + addr, addr + size, shadow_begin, shadow_end); + + if (!data_mapped) { + // First call maps data+bss. + if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, "shadow")) + Die(); + } else { + VPrintf(2, "ctx->mapped_shadow_{begin,end} = (0x%zx-0x%zx)\n", + ctx->mapped_shadow_begin, ctx->mapped_shadow_end); + // Second and subsequent calls map heap. + if (shadow_end <= ctx->mapped_shadow_end) + return; + if (!ctx->mapped_shadow_begin || ctx->mapped_shadow_begin > shadow_begin) + ctx->mapped_shadow_begin = shadow_begin; + if (shadow_begin < ctx->mapped_shadow_end) + shadow_begin = ctx->mapped_shadow_end; + VPrintf(2, "MapShadow begin/end = (0x%zx-0x%zx)\n", + shadow_begin, shadow_end); + if (!MmapFixedSuperNoReserve(shadow_begin, shadow_end - shadow_begin, + "shadow")) + Die(); + ctx->mapped_shadow_end = shadow_end; + } +#endif // Meta shadow is 2:1, so tread carefully. - static bool data_mapped = false; static uptr mapped_meta_end = 0; uptr meta_begin = (uptr)MemToMeta(addr); uptr meta_end = (uptr)MemToMeta(addr + size); @@ -587,8 +633,7 @@ void MapShadow(uptr addr, uptr size) { // Windows wants 64K alignment. meta_begin = RoundDownTo(meta_begin, 64 << 10); meta_end = RoundUpTo(meta_end, 64 << 10); - if (meta_end <= mapped_meta_end) - return; + CHECK_GT(meta_end, mapped_meta_end); if (meta_begin < mapped_meta_end) meta_begin = mapped_meta_end; if (!MmapFixedSuperNoReserve(meta_begin, meta_end - meta_begin, @@ -651,9 +696,6 @@ void Initialize(ThreadState *thr) { __tsan::InitializePlatformEarly(); #if !SANITIZER_GO - // Re-exec ourselves if we need to set additional env or command line args. - MaybeReexec(); - InitializeAllocator(); ReplaceSystemMalloc(); #endif @@ -722,8 +764,10 @@ void MaybeSpawnBackgroundThread() { int Finalize(ThreadState *thr) { bool failed = false; +#if !SANITIZER_GO if (common_flags()->print_module_map == 1) DumpProcessMap(); +#endif if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1) internal_usleep(u64(flags()->atexit_sleep_ms) * 1000); @@ -952,6 +996,15 @@ void TraceSwitchPartImpl(ThreadState* thr) { TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0, d.addr, d.stack_id); } + // Callers of TraceSwitchPart expect that TraceAcquire will always succeed + // after the call. It's possible that TryTraceFunc/TraceMutexLock above + // filled the trace part exactly up to the TracePart::kAlignment gap + // and the next TraceAcquire won't succeed. Skip the gap to avoid that. + EventFunc *ev; + if (!TraceAcquire(thr, &ev)) { + CHECK(TraceSkipGap(thr)); + CHECK(TraceAcquire(thr, &ev)); + } { Lock lock(&ctx->slot_mtx); // There is a small chance that the slot may be not queued at this point. diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h index b472c0f77df..e1e121e2ee0 100644 --- a/libsanitizer/tsan/tsan_rtl.h +++ b/libsanitizer/tsan/tsan_rtl.h @@ -235,7 +235,7 @@ struct ThreadState { } ALIGNED(SANITIZER_CACHE_LINE_SIZE); #if !SANITIZER_GO -#if SANITIZER_MAC || SANITIZER_ANDROID +#if SANITIZER_APPLE || SANITIZER_ANDROID ThreadState *cur_thread(); void set_cur_thread(ThreadState *thr); void cur_thread_finalize(); @@ -256,7 +256,7 @@ inline void set_cur_thread(ThreadState *thr) { reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr; } inline void cur_thread_finalize() { } -# endif // SANITIZER_MAC || SANITIZER_ANDROID +# endif // SANITIZER_APPLE || SANITIZER_ANDROID #endif // SANITIZER_GO class ThreadContext final : public ThreadContextBase { @@ -314,9 +314,43 @@ struct Context { ThreadRegistry thread_registry; + // This is used to prevent a very unlikely but very pathological behavior. + // Since memory access handling is not synchronized with DoReset, + // a thread running concurrently with DoReset can leave a bogus shadow value + // that will be later falsely detected as a race. For such false races + // RestoreStack will return false and we will not report it. + // However, consider that a thread leaves a whole lot of such bogus values + // and these values are later read by a whole lot of threads. + // This will cause massive amounts of ReportRace calls and lots of + // serialization. In very pathological cases the resulting slowdown + // can be >100x. This is very unlikely, but it was presumably observed + // in practice: https://github.com/google/sanitizers/issues/1552 + // If this happens, previous access sid+epoch will be the same for all of + // these false races b/c if the thread will try to increment epoch, it will + // notice that DoReset has happened and will stop producing bogus shadow + // values. So, last_spurious_race is used to remember the last sid+epoch + // for which RestoreStack returned false. Then it is used to filter out + // races with the same sid+epoch very early and quickly. + // It is of course possible that multiple threads left multiple bogus shadow + // values and all of them are read by lots of threads at the same time. + // In such case last_spurious_race will only be able to deduplicate a few + // races from one thread, then few from another and so on. An alternative + // would be to hold an array of such sid+epoch, but we consider such scenario + // as even less likely. + // Note: this can lead to some rare false negatives as well: + // 1. When a legit access with the same sid+epoch participates in a race + // as the "previous" memory access, it will be wrongly filtered out. + // 2. When RestoreStack returns false for a legit memory access because it + // was already evicted from the thread trace, we will still remember it in + // last_spurious_race. Then if there is another racing memory access from + // the same thread that happened in the same epoch, but was stored in the + // next thread trace part (which is still preserved in the thread trace), + // we will also wrongly filter it out while RestoreStack would actually + // succeed for that second memory access. + RawShadow last_spurious_race; + Mutex racy_mtx; Vector<RacyStacks> racy_stacks; - Vector<RacyAddress> racy_addresses; // Number of fired suppressions may be large enough. Mutex fired_suppressions_mtx; InternalMmapVector<FiredSuppression> fired_suppressions; @@ -338,6 +372,10 @@ struct Context { uptr trace_part_total_allocated SANITIZER_GUARDED_BY(slot_mtx); uptr trace_part_recycle_finished SANITIZER_GUARDED_BY(slot_mtx); uptr trace_part_finished_excess SANITIZER_GUARDED_BY(slot_mtx); +#if SANITIZER_GO + uptr mapped_shadow_begin; + uptr mapped_shadow_end; +#endif }; extern Context *ctx; // The one and the only global runtime context. diff --git a/libsanitizer/tsan/tsan_rtl_access.cpp b/libsanitizer/tsan/tsan_rtl_access.cpp index 7d771bfaad7..8b20984a010 100644 --- a/libsanitizer/tsan/tsan_rtl_access.cpp +++ b/libsanitizer/tsan/tsan_rtl_access.cpp @@ -145,15 +145,6 @@ void TraceTime(ThreadState* thr) { TraceEvent(thr, ev); } -ALWAYS_INLINE RawShadow LoadShadow(RawShadow* p) { - return static_cast<RawShadow>( - atomic_load((atomic_uint32_t*)p, memory_order_relaxed)); -} - -ALWAYS_INLINE void StoreShadow(RawShadow* sp, RawShadow s) { - atomic_store((atomic_uint32_t*)sp, static_cast<u32>(s), memory_order_relaxed); -} - NOINLINE void DoReportRace(ThreadState* thr, RawShadow* shadow_mem, Shadow cur, Shadow old, AccessType typ) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { diff --git a/libsanitizer/tsan/tsan_rtl_ppc64.S b/libsanitizer/tsan/tsan_rtl_ppc64.S index 9e533a71a9c..8285e21aa1e 100644 --- a/libsanitizer/tsan/tsan_rtl_ppc64.S +++ b/libsanitizer/tsan/tsan_rtl_ppc64.S @@ -1,6 +1,5 @@ #include "tsan_ppc_regs.h" - .machine altivec .section .text .hidden __tsan_setjmp .globl _setjmp diff --git a/libsanitizer/tsan/tsan_rtl_report.cpp b/libsanitizer/tsan/tsan_rtl_report.cpp index 4cf8816489d..c2cff60e2da 100644 --- a/libsanitizer/tsan/tsan_rtl_report.cpp +++ b/libsanitizer/tsan/tsan_rtl_report.cpp @@ -281,16 +281,16 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) { int fd = -1; Tid creat_tid = kInvalidTid; StackID creat_stack = 0; - if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) { + bool closed = false; + if (FdLocation(addr, &fd, &creat_tid, &creat_stack, &closed)) { auto *loc = New<ReportLocation>(); loc->type = ReportLocationFD; + loc->fd_closed = closed; loc->fd = fd; loc->tid = creat_tid; loc->stack = SymbolizeStackId(creat_stack); rep_->locs.PushBack(loc); - ThreadContext *tctx = FindThreadByTidLocked(creat_tid); - if (tctx) - AddThread(tctx); + AddThread(creat_tid); return; } MBlock *b = 0; @@ -312,8 +312,7 @@ void ScopedReportBase::AddLocation(uptr addr, uptr size) { loc->tid = b->tid; loc->stack = SymbolizeStackId(b->stk); rep_->locs.PushBack(loc); - if (ThreadContext *tctx = FindThreadByTidLocked(b->tid)) - AddThread(tctx); + AddThread(b->tid); return; } bool is_stack = false; @@ -629,35 +628,6 @@ static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2]) { return false; } -static bool FindRacyAddress(const RacyAddress &ra0) { - for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) { - RacyAddress ra2 = ctx->racy_addresses[i]; - uptr maxbeg = max(ra0.addr_min, ra2.addr_min); - uptr minend = min(ra0.addr_max, ra2.addr_max); - if (maxbeg < minend) { - VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n"); - return true; - } - } - return false; -} - -static bool HandleRacyAddress(ThreadState *thr, uptr addr_min, uptr addr_max) { - if (!flags()->suppress_equal_addresses) - return false; - RacyAddress ra0 = {addr_min, addr_max}; - { - ReadLock lock(&ctx->racy_mtx); - if (FindRacyAddress(ra0)) - return true; - } - Lock lock(&ctx->racy_mtx); - if (FindRacyAddress(ra0)) - return true; - ctx->racy_addresses.PushBack(ra0); - return false; -} - bool OutputReport(ThreadState *thr, const ScopedReport &srep) { // These should have been checked in ShouldReport. // It's too late to check them here, we have already taken locks. @@ -730,6 +700,11 @@ static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) { return false; } +static bool SpuriousRace(Shadow old) { + Shadow last(LoadShadow(&ctx->last_spurious_race)); + return last.sid() == old.sid() && last.epoch() == old.epoch(); +} + void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old, AccessType typ0) { CheckedMutex::CheckNoLocks(); @@ -750,6 +725,8 @@ void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old, ((typ0 & kAccessAtomic) || (typ1 & kAccessAtomic)) && !(typ0 & kAccessFree) && !(typ1 & kAccessFree)) return; + if (SpuriousRace(old)) + return; const uptr kMop = 2; Shadow s[kMop] = {cur, old}; @@ -761,8 +738,6 @@ void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old, uptr addr_max = max(end0, end1); if (IsExpectedReport(addr_min, addr_max - addr_min)) return; - if (HandleRacyAddress(thr, addr_min, addr_max)) - return; ReportType rep_typ = ReportTypeRace; if ((typ0 & kAccessVptr) && (typ1 & kAccessFree)) @@ -791,9 +766,13 @@ void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old, Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx); ThreadRegistryLock l0(&ctx->thread_registry); Lock slots_lock(&ctx->slot_mtx); + if (SpuriousRace(old)) + return; if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1, - size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) + size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) { + StoreShadow(&ctx->last_spurious_race, old.raw()); return; + } if (IsFiredSuppression(ctx, rep_typ, traces[1])) return; diff --git a/libsanitizer/tsan/tsan_rtl_thread.cpp b/libsanitizer/tsan/tsan_rtl_thread.cpp index 86c8b3764cc..77488f84328 100644 --- a/libsanitizer/tsan/tsan_rtl_thread.cpp +++ b/libsanitizer/tsan/tsan_rtl_thread.cpp @@ -53,7 +53,7 @@ static void CollectThreadLeaks(ThreadContextBase *tctx_base, void *arg) { // Disabled on Mac because lldb test TestTsanBasic fails: // https://reviews.llvm.org/D112603#3163158 -#if !SANITIZER_GO && !SANITIZER_MAC +#if !SANITIZER_GO && !SANITIZER_APPLE static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) { if (tctx->tid == kMainTid) { Printf("ThreadSanitizer: main thread finished with ignores enabled\n"); diff --git a/libsanitizer/tsan/tsan_shadow.h b/libsanitizer/tsan/tsan_shadow.h index b222acf9e6c..6b8114ef513 100644 --- a/libsanitizer/tsan/tsan_shadow.h +++ b/libsanitizer/tsan/tsan_shadow.h @@ -178,6 +178,16 @@ class Shadow { static_assert(sizeof(Shadow) == kShadowSize, "bad Shadow size"); +ALWAYS_INLINE RawShadow LoadShadow(RawShadow *p) { + return static_cast<RawShadow>( + atomic_load((atomic_uint32_t *)p, memory_order_relaxed)); +} + +ALWAYS_INLINE void StoreShadow(RawShadow *sp, RawShadow s) { + atomic_store((atomic_uint32_t *)sp, static_cast<u32>(s), + memory_order_relaxed); +} + } // namespace __tsan #endif diff --git a/libsanitizer/ubsan/ubsan_flags.cpp b/libsanitizer/ubsan/ubsan_flags.cpp index 9a66bd37518..25cefd46ce2 100644 --- a/libsanitizer/ubsan/ubsan_flags.cpp +++ b/libsanitizer/ubsan/ubsan_flags.cpp @@ -50,7 +50,6 @@ void InitializeFlags() { { CommonFlags cf; cf.CopyFrom(*common_flags()); - cf.print_summary = false; cf.external_symbolizer_path = GetFlag("UBSAN_SYMBOLIZER_PATH"); OverrideCommonFlags(cf); } diff --git a/libsanitizer/ubsan/ubsan_handlers.cpp b/libsanitizer/ubsan/ubsan_handlers.cpp index 2184625aa6e..410292a0d53 100644 --- a/libsanitizer/ubsan/ubsan_handlers.cpp +++ b/libsanitizer/ubsan/ubsan_handlers.cpp @@ -76,7 +76,7 @@ enum TypeCheckKind { TCK_DynamicOperation }; -const char *TypeCheckKinds[] = { +extern const char *const TypeCheckKinds[] = { "load of", "store to", "reference binding to", "member access within", "member call on", "constructor call on", "downcast of", "downcast of", "upcast of", "cast to virtual base of", "_Nonnull binding to", @@ -894,21 +894,6 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, } // namespace __ubsan -void __ubsan::__ubsan_handle_cfi_bad_icall(CFIBadIcallData *CallData, - ValueHandle Function) { - GET_REPORT_OPTIONS(false); - CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type}; - handleCFIBadIcall(&Data, Function, Opts); -} - -void __ubsan::__ubsan_handle_cfi_bad_icall_abort(CFIBadIcallData *CallData, - ValueHandle Function) { - GET_REPORT_OPTIONS(true); - CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type}; - handleCFIBadIcall(&Data, Function, Opts); - Die(); -} - void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data, ValueHandle Value, uptr ValidVtable) { diff --git a/libsanitizer/ubsan/ubsan_handlers.h b/libsanitizer/ubsan/ubsan_handlers.h index 9f412353fc0..219fb15de55 100644 --- a/libsanitizer/ubsan/ubsan_handlers.h +++ b/libsanitizer/ubsan/ubsan_handlers.h @@ -215,20 +215,12 @@ enum CFITypeCheckKind : unsigned char { CFITCK_VMFCall, }; -struct CFIBadIcallData { - SourceLocation Loc; - const TypeDescriptor &Type; -}; - struct CFICheckFailData { CFITypeCheckKind CheckKind; SourceLocation Loc; const TypeDescriptor &Type; }; -/// \brief Handle control flow integrity failure for indirect function calls. -RECOVERABLE(cfi_bad_icall, CFIBadIcallData *Data, ValueHandle Function) - /// \brief Handle control flow integrity failures. RECOVERABLE(cfi_check_fail, CFICheckFailData *Data, ValueHandle Function, uptr VtableIsValid) diff --git a/libsanitizer/ubsan/ubsan_handlers_cxx.cpp b/libsanitizer/ubsan/ubsan_handlers_cxx.cpp index 2a6d558de03..0317a3d1428 100644 --- a/libsanitizer/ubsan/ubsan_handlers_cxx.cpp +++ b/libsanitizer/ubsan/ubsan_handlers_cxx.cpp @@ -26,7 +26,7 @@ using namespace __sanitizer; using namespace __ubsan; namespace __ubsan { - extern const char *TypeCheckKinds[]; + extern const char *const TypeCheckKinds[]; } // Returns true if UBSan has printed an error report. diff --git a/libsanitizer/ubsan/ubsan_platform.h b/libsanitizer/ubsan/ubsan_platform.h index ad3e883f0f3..d2cc2e10bd2 100644 --- a/libsanitizer/ubsan/ubsan_platform.h +++ b/libsanitizer/ubsan/ubsan_platform.h @@ -12,7 +12,6 @@ #ifndef UBSAN_PLATFORM_H #define UBSAN_PLATFORM_H -#ifndef CAN_SANITIZE_UB // Other platforms should be easy to add, and probably work as-is. #if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || \ defined(__NetBSD__) || defined(__DragonFly__) || \ @@ -22,6 +21,5 @@ #else # define CAN_SANITIZE_UB 0 #endif -#endif //CAN_SANITIZE_UB #endif diff --git a/libsanitizer/ubsan/ubsan_value.cpp b/libsanitizer/ubsan/ubsan_value.cpp index 40042bf3a90..dc61e5b939d 100644 --- a/libsanitizer/ubsan/ubsan_value.cpp +++ b/libsanitizer/ubsan/ubsan_value.cpp @@ -18,9 +18,7 @@ #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_mutex.h" -// TODO(dliew): Prefer '__APPLE__' here over 'SANITIZER_MAC', as the latter is -// unclear. rdar://58124919 tracks using a more obviously portable guard. -#if defined(__APPLE__) +#if SANITIZER_APPLE #include <dlfcn.h> #endif @@ -29,7 +27,7 @@ using namespace __ubsan; typedef const char *(*ObjCGetClassNameTy)(void *); const char *__ubsan::getObjCClassName(ValueHandle Pointer) { -#if defined(__APPLE__) +#if SANITIZER_APPLE // We need to query the ObjC runtime for some information, but do not want // to introduce a static dependency from the ubsan runtime onto ObjC. Try to // grab a handle to the ObjC runtime used by the process. |