diff options
Diffstat (limited to 'libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc')
| -rw-r--r-- | libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc | 753 |
1 files changed, 613 insertions, 140 deletions
diff --git a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc index 214e8a98d0f..108a6163d19 100644 --- a/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc +++ b/libsanitizer/sanitizer_common/sanitizer_coverage_libcdep.cc @@ -10,18 +10,24 @@ // // Compiler instrumentation: // For every interesting basic block the compiler injects the following code: -// if (*Guard) { -// __sanitizer_cov(); -// *Guard = 1; +// if (Guard < 0) { +// __sanitizer_cov(&Guard); // } +// At the module start up time __sanitizer_cov_module_init sets the guards +// to consecutive negative numbers (-1, -2, -3, ...). // It's fine to call __sanitizer_cov more than once for a given block. // // Run-time: // - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC). +// and atomically set Guard to -Guard. // - __sanitizer_cov_dump: dump the coverage data to disk. // For every module of the current process that has coverage data -// this will create a file module_name.PID.sancov. The file format is simple: -// it's just a sorted sequence of 4-byte offsets in the module. +// this will create a file module_name.PID.sancov. +// +// The file format is simple: the first 8 bytes is the magic, +// one of 0xC0BFFFFFFFFFFF64 and 0xC0BFFFFFFFFFFF32. The last byte of the +// magic defines the size of the following offsets. +// The rest of the data is the offsets in the module. // // Eventually, this coverage implementation should be obsoleted by a more // powerful general purpose Clang/LLVM coverage instrumentation. @@ -39,7 +45,12 @@ #include "sanitizer_symbolizer.h" #include "sanitizer_flags.h" -atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once. +static const u64 kMagic64 = 0xC0BFFFFFFFFFFF64ULL; +static const u64 kMagic32 = 0xC0BFFFFFFFFFFF32ULL; + +static atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once. + +static atomic_uintptr_t coverage_counter; // pc_array is the array containing the covered PCs. // To make the pc_array thread- and async-signal-safe it has to be large enough. @@ -50,29 +61,55 @@ atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once. // dump current memory layout to another file. static bool cov_sandboxed = false; -static int cov_fd = kInvalidFd; +static fd_t cov_fd = kInvalidFd; static unsigned int cov_max_block_size = 0; +static bool coverage_enabled = false; +static const char *coverage_dir; namespace __sanitizer { class CoverageData { public: void Init(); + void Enable(); + void Disable(); + void ReInit(); void BeforeFork(); void AfterFork(int child_pid); void Extend(uptr npcs); - void Add(uptr pc); + void Add(uptr pc, u32 *guard); void IndirCall(uptr caller, uptr callee, uptr callee_cache[], uptr cache_size); void DumpCallerCalleePairs(); + void DumpTrace(); + void DumpAsBitSet(); + void DumpCounters(); + void DumpOffsets(); + void DumpAll(); + + ALWAYS_INLINE + void TraceBasicBlock(s32 *id); + + void InitializeGuardArray(s32 *guards); + void InitializeGuards(s32 *guards, uptr n, const char *module_name, + uptr caller_pc); + void InitializeCounters(u8 *counters, uptr n); + void ReinitializeGuards(); + uptr GetNumberOf8bitCounters(); + uptr Update8bitCounterBitsetAndClearCounters(u8 *bitset); uptr *data(); uptr size(); private: + void DirectOpen(); + void UpdateModuleNameVec(uptr caller_pc, uptr range_beg, uptr range_end); + // Maximal size pc array may ever grow. // We MmapNoReserve this space to ensure that the array is contiguous. - static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27); + static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64( + 1 << (SANITIZER_ANDROID ? 24 : (SANITIZER_WINDOWS ? 27 : 26)), + 1 << 27); // The amount file mapping for the pc array is grown by. static const uptr kPcArrayMmapSize = 64 * 1024; @@ -86,7 +123,27 @@ class CoverageData { // Current file mapped size of the pc array. uptr pc_array_mapped_size; // Descriptor of the file mapped pc array. - int pc_fd; + fd_t pc_fd; + + // Vector of coverage guard arrays, protected by mu. + InternalMmapVectorNoCtor<s32*> guard_array_vec; + + struct NamedPcRange { + const char *copied_module_name; + uptr beg, end; // elements [beg,end) in pc_array. + }; + + // Vector of module and compilation unit pc ranges. + InternalMmapVectorNoCtor<NamedPcRange> comp_unit_name_vec; + InternalMmapVectorNoCtor<NamedPcRange> module_name_vec; + + struct CounterAndSize { + u8 *counters; + uptr n; + }; + + InternalMmapVectorNoCtor<CounterAndSize> counters_vec; + uptr num_8bit_counters; // Caller-Callee (cc) array, size and current index. static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24); @@ -94,59 +151,131 @@ class CoverageData { atomic_uintptr_t cc_array_index; atomic_uintptr_t cc_array_size; + // Tracing event array, size and current pointer. + // We record all events (basic block entries) in a global buffer of u32 + // values. Each such value is the index in pc_array. + // So far the tracing is highly experimental: + // - not thread-safe; + // - does not support long traces; + // - not tuned for performance. + static const uptr kTrEventArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 30); + u32 *tr_event_array; + uptr tr_event_array_size; + u32 *tr_event_pointer; + static const uptr kTrPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27); StaticSpinMutex mu; - - void DirectOpen(); - void ReInit(); }; static CoverageData coverage_data; +void CovUpdateMapping(const char *path, uptr caller_pc = 0); + void CoverageData::DirectOpen() { - InternalScopedString path(1024); + InternalScopedString path(kMaxPathLength); internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw", - common_flags()->coverage_dir, internal_getpid()); - pc_fd = OpenFile(path.data(), true); - if (internal_iserror(pc_fd)) { - Report(" Coverage: failed to open %s for writing\n", path.data()); + coverage_dir, internal_getpid()); + pc_fd = OpenFile(path.data(), RdWr); + if (pc_fd == kInvalidFd) { + Report("Coverage: failed to open %s for reading/writing\n", path.data()); Die(); } pc_array_mapped_size = 0; - CovUpdateMapping(); + CovUpdateMapping(coverage_dir); } void CoverageData::Init() { + pc_fd = kInvalidFd; +} + +void CoverageData::Enable() { + if (pc_array) + return; pc_array = reinterpret_cast<uptr *>( MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit")); - pc_fd = kInvalidFd; + atomic_store(&pc_array_index, 0, memory_order_relaxed); if (common_flags()->coverage_direct) { atomic_store(&pc_array_size, 0, memory_order_relaxed); - atomic_store(&pc_array_index, 0, memory_order_relaxed); } else { atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed); - atomic_store(&pc_array_index, 0, memory_order_relaxed); } cc_array = reinterpret_cast<uptr **>(MmapNoReserveOrDie( sizeof(uptr *) * kCcArrayMaxSize, "CovInit::cc_array")); atomic_store(&cc_array_size, kCcArrayMaxSize, memory_order_relaxed); atomic_store(&cc_array_index, 0, memory_order_relaxed); + + // Allocate tr_event_array with a guard page at the end. + tr_event_array = reinterpret_cast<u32 *>(MmapNoReserveOrDie( + sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(), + "CovInit::tr_event_array")); + MprotectNoAccess( + reinterpret_cast<uptr>(&tr_event_array[kTrEventArrayMaxSize]), + GetMmapGranularity()); + tr_event_array_size = kTrEventArrayMaxSize; + tr_event_pointer = tr_event_array; + + num_8bit_counters = 0; +} + +void CoverageData::InitializeGuardArray(s32 *guards) { + Enable(); // Make sure coverage is enabled at this point. + s32 n = guards[0]; + for (s32 j = 1; j <= n; j++) { + uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed); + guards[j] = -static_cast<s32>(idx + 1); + } +} + +void CoverageData::Disable() { + if (pc_array) { + UnmapOrDie(pc_array, sizeof(uptr) * kPcArrayMaxSize); + pc_array = nullptr; + } + if (cc_array) { + UnmapOrDie(cc_array, sizeof(uptr *) * kCcArrayMaxSize); + cc_array = nullptr; + } + if (tr_event_array) { + UnmapOrDie(tr_event_array, + sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + + GetMmapGranularity()); + tr_event_array = nullptr; + tr_event_pointer = nullptr; + } + if (pc_fd != kInvalidFd) { + CloseFile(pc_fd); + pc_fd = kInvalidFd; + } +} + +void CoverageData::ReinitializeGuards() { + // Assuming single thread. + atomic_store(&pc_array_index, 0, memory_order_relaxed); + for (uptr i = 0; i < guard_array_vec.size(); i++) + InitializeGuardArray(guard_array_vec[i]); } void CoverageData::ReInit() { - internal_munmap(pc_array, sizeof(uptr) * kPcArrayMaxSize); - if (pc_fd != kInvalidFd) internal_close(pc_fd); - if (common_flags()->coverage_direct) { - // In memory-mapped mode we must extend the new file to the known array - // size. - uptr size = atomic_load(&pc_array_size, memory_order_relaxed); - Init(); - if (size) Extend(size); - } else { - Init(); + Disable(); + if (coverage_enabled) { + if (common_flags()->coverage_direct) { + // In memory-mapped mode we must extend the new file to the known array + // size. + uptr size = atomic_load(&pc_array_size, memory_order_relaxed); + uptr npcs = size / sizeof(uptr); + Enable(); + if (size) Extend(npcs); + if (coverage_enabled) CovUpdateMapping(coverage_dir); + } else { + Enable(); + } } + // Re-initialize the guards. + // We are single-threaded now, no need to grab any lock. + CHECK_EQ(atomic_load(&pc_array_index, memory_order_relaxed), 0); + ReinitializeGuards(); } void CoverageData::BeforeFork() { @@ -164,15 +293,16 @@ void CoverageData::Extend(uptr npcs) { if (!common_flags()->coverage_direct) return; SpinMutexLock l(&mu); - if (pc_fd == kInvalidFd) DirectOpen(); - CHECK_NE(pc_fd, kInvalidFd); - uptr size = atomic_load(&pc_array_size, memory_order_relaxed); size += npcs * sizeof(uptr); - if (size > pc_array_mapped_size) { + if (coverage_enabled && size > pc_array_mapped_size) { + if (pc_fd == kInvalidFd) DirectOpen(); + CHECK_NE(pc_fd, kInvalidFd); + uptr new_mapped_size = pc_array_mapped_size; while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize; + CHECK_LE(new_mapped_size, sizeof(uptr) * kPcArrayMaxSize); // Extend the file and map the new space at the end of pc_array. uptr res = internal_ftruncate(pc_fd, new_mapped_size); @@ -181,24 +311,100 @@ void CoverageData::Extend(uptr npcs) { Printf("failed to extend raw coverage file: %d\n", err); Die(); } - void *p = MapWritableFileToMemory(pc_array + pc_array_mapped_size, + + uptr next_map_base = ((uptr)pc_array) + pc_array_mapped_size; + void *p = MapWritableFileToMemory((void *)next_map_base, new_mapped_size - pc_array_mapped_size, pc_fd, pc_array_mapped_size); - CHECK_EQ(p, pc_array + pc_array_mapped_size); + CHECK_EQ((uptr)p, next_map_base); pc_array_mapped_size = new_mapped_size; } atomic_store(&pc_array_size, size, memory_order_release); } -// Simply add the pc into the vector under lock. If the function is called more -// than once for a given PC it will be inserted multiple times, which is fine. -void CoverageData::Add(uptr pc) { +void CoverageData::InitializeCounters(u8 *counters, uptr n) { + if (!counters) return; + CHECK_EQ(reinterpret_cast<uptr>(counters) % 16, 0); + n = RoundUpTo(n, 16); // The compiler must ensure that counters is 16-aligned. + SpinMutexLock l(&mu); + counters_vec.push_back({counters, n}); + num_8bit_counters += n; +} + +void CoverageData::UpdateModuleNameVec(uptr caller_pc, uptr range_beg, + uptr range_end) { + auto sym = Symbolizer::GetOrInit(); + if (!sym) + return; + const char *module_name = sym->GetModuleNameForPc(caller_pc); + if (!module_name) return; + if (module_name_vec.empty() || + module_name_vec.back().copied_module_name != module_name) + module_name_vec.push_back({module_name, range_beg, range_end}); + else + module_name_vec.back().end = range_end; +} + +void CoverageData::InitializeGuards(s32 *guards, uptr n, + const char *comp_unit_name, + uptr caller_pc) { + // The array 'guards' has n+1 elements, we use the element zero + // to store 'n'. + CHECK_LT(n, 1 << 30); + guards[0] = static_cast<s32>(n); + InitializeGuardArray(guards); + SpinMutexLock l(&mu); + uptr range_end = atomic_load(&pc_array_index, memory_order_relaxed); + uptr range_beg = range_end - n; + comp_unit_name_vec.push_back({comp_unit_name, range_beg, range_end}); + guard_array_vec.push_back(guards); + UpdateModuleNameVec(caller_pc, range_beg, range_end); +} + +static const uptr kBundleCounterBits = 16; + +// When coverage_order_pcs==true and SANITIZER_WORDSIZE==64 +// we insert the global counter into the first 16 bits of the PC. +uptr BundlePcAndCounter(uptr pc, uptr counter) { + if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) + return pc; + static const uptr kMaxCounter = (1 << kBundleCounterBits) - 1; + if (counter > kMaxCounter) + counter = kMaxCounter; + CHECK_EQ(0, pc >> (SANITIZER_WORDSIZE - kBundleCounterBits)); + return pc | (counter << (SANITIZER_WORDSIZE - kBundleCounterBits)); +} + +uptr UnbundlePc(uptr bundle) { + if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) + return bundle; + return (bundle << kBundleCounterBits) >> kBundleCounterBits; +} + +uptr UnbundleCounter(uptr bundle) { + if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) + return 0; + return bundle >> (SANITIZER_WORDSIZE - kBundleCounterBits); +} + +// If guard is negative, atomically set it to -guard and store the PC in +// pc_array. +void CoverageData::Add(uptr pc, u32 *guard) { + atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard); + s32 guard_value = atomic_load(atomic_guard, memory_order_relaxed); + if (guard_value >= 0) return; + + atomic_store(atomic_guard, -guard_value, memory_order_relaxed); if (!pc_array) return; - uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed); + + uptr idx = -guard_value - 1; + if (idx >= atomic_load(&pc_array_index, memory_order_acquire)) + return; // May happen after fork when pc_array_index becomes 0. CHECK_LT(idx * sizeof(uptr), atomic_load(&pc_array_size, memory_order_acquire)); - pc_array[idx] = pc; + uptr counter = atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed); + pc_array[idx] = BundlePcAndCounter(pc, counter); } // Registers a pair caller=>callee. @@ -226,13 +432,73 @@ void CoverageData::IndirCall(uptr caller, uptr callee, uptr callee_cache[], for (uptr i = 2; i < cache_size; i++) { uptr was = 0; if (atomic_compare_exchange_strong(&atomic_callee_cache[i], &was, callee, - memory_order_seq_cst)) + memory_order_seq_cst)) { + atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed); return; + } if (was == callee) // Already have this callee. return; } } +uptr CoverageData::GetNumberOf8bitCounters() { + return num_8bit_counters; +} + +// Map every 8bit counter to a 8-bit bitset and clear the counter. +uptr CoverageData::Update8bitCounterBitsetAndClearCounters(u8 *bitset) { + uptr num_new_bits = 0; + uptr cur = 0; + // For better speed we map 8 counters to 8 bytes of bitset at once. + static const uptr kBatchSize = 8; + CHECK_EQ(reinterpret_cast<uptr>(bitset) % kBatchSize, 0); + for (uptr i = 0, len = counters_vec.size(); i < len; i++) { + u8 *c = counters_vec[i].counters; + uptr n = counters_vec[i].n; + CHECK_EQ(n % 16, 0); + CHECK_EQ(cur % kBatchSize, 0); + CHECK_EQ(reinterpret_cast<uptr>(c) % kBatchSize, 0); + if (!bitset) { + internal_bzero_aligned16(c, n); + cur += n; + continue; + } + for (uptr j = 0; j < n; j += kBatchSize, cur += kBatchSize) { + CHECK_LT(cur, num_8bit_counters); + u64 *pc64 = reinterpret_cast<u64*>(c + j); + u64 *pb64 = reinterpret_cast<u64*>(bitset + cur); + u64 c64 = *pc64; + u64 old_bits_64 = *pb64; + u64 new_bits_64 = old_bits_64; + if (c64) { + *pc64 = 0; + for (uptr k = 0; k < kBatchSize; k++) { + u64 x = (c64 >> (8 * k)) & 0xff; + if (x) { + u64 bit = 0; + /**/ if (x >= 128) bit = 128; + else if (x >= 32) bit = 64; + else if (x >= 16) bit = 32; + else if (x >= 8) bit = 16; + else if (x >= 4) bit = 8; + else if (x >= 3) bit = 4; + else if (x >= 2) bit = 2; + else if (x >= 1) bit = 1; + u64 mask = bit << (8 * k); + if (!(new_bits_64 & mask)) { + num_new_bits++; + new_bits_64 |= mask; + } + } + } + *pb64 = new_bits_64; + } + } + } + CHECK_EQ(cur, num_8bit_counters); + return num_new_bits; +} + uptr *CoverageData::data() { return pc_array; } @@ -251,15 +517,15 @@ struct CovHeader { static void CovWritePacked(int pid, const char *module, const void *blob, unsigned int blob_size) { - if (cov_fd < 0) return; + if (cov_fd == kInvalidFd) return; unsigned module_name_length = internal_strlen(module); CovHeader header = {pid, module_name_length, blob_size}; if (cov_max_block_size == 0) { // Writing to a file. Just go ahead. - internal_write(cov_fd, &header, sizeof(header)); - internal_write(cov_fd, module, module_name_length); - internal_write(cov_fd, blob, blob_size); + WriteToFile(cov_fd, &header, sizeof(header)); + WriteToFile(cov_fd, module, module_name_length); + WriteToFile(cov_fd, blob, blob_size); } else { // Writing to a socket. We want to split the data into appropriately sized // blocks. @@ -275,15 +541,14 @@ static void CovWritePacked(int pid, const char *module, const void *blob, internal_memcpy(block_pos, module, module_name_length); block_pos += module_name_length; char *block_data_begin = block_pos; - char *blob_pos = (char *)blob; + const char *blob_pos = (const char *)blob; while (blob_size > 0) { unsigned int payload_size = Min(blob_size, max_payload_size); blob_size -= payload_size; internal_memcpy(block_data_begin, blob_pos, payload_size); blob_pos += payload_size; ((CovHeader *)block.data())->data_length = payload_size; - internal_write(cov_fd, block.data(), - header_size_with_module + payload_size); + WriteToFile(cov_fd, block.data(), header_size_with_module + payload_size); } } } @@ -292,29 +557,77 @@ static void CovWritePacked(int pid, const char *module, const void *blob, // If packed = true and name == 0: <pid>.<sancov>.<packed>. // If packed = true and name != 0: <name>.<sancov>.<packed> (name is // user-supplied). -static int CovOpenFile(bool packed, const char* name) { - InternalScopedBuffer<char> path(1024); +static fd_t CovOpenFile(InternalScopedString *path, bool packed, + const char *name, const char *extension = "sancov") { + path->clear(); if (!packed) { CHECK(name); - internal_snprintf((char *)path.data(), path.size(), "%s/%s.%zd.sancov", - common_flags()->coverage_dir, name, internal_getpid()); + path->append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(), + extension); } else { if (!name) - internal_snprintf((char *)path.data(), path.size(), - "%s/%zd.sancov.packed", common_flags()->coverage_dir, - internal_getpid()); + path->append("%s/%zd.%s.packed", coverage_dir, internal_getpid(), + extension); else - internal_snprintf((char *)path.data(), path.size(), "%s/%s.sancov.packed", - common_flags()->coverage_dir, name); - } - uptr fd = OpenFile(path.data(), true); - if (internal_iserror(fd)) { - Report(" SanitizerCoverage: failed to open %s for writing\n", path.data()); - return -1; + path->append("%s/%s.%s.packed", coverage_dir, name, extension); } + error_t err; + fd_t fd = OpenFile(path->data(), WrOnly, &err); + if (fd == kInvalidFd) + Report("SanitizerCoverage: failed to open %s for writing (reason: %d)\n", + path->data(), err); return fd; } +// Dump trace PCs and trace events into two separate files. +void CoverageData::DumpTrace() { + uptr max_idx = tr_event_pointer - tr_event_array; + if (!max_idx) return; + auto sym = Symbolizer::GetOrInit(); + if (!sym) + return; + InternalScopedString out(32 << 20); + for (uptr i = 0, n = size(); i < n; i++) { + const char *module_name = "<unknown>"; + uptr module_address = 0; + sym->GetModuleNameAndOffsetForPC(UnbundlePc(pc_array[i]), &module_name, + &module_address); + out.append("%s 0x%zx\n", module_name, module_address); + } + InternalScopedString path(kMaxPathLength); + fd_t fd = CovOpenFile(&path, false, "trace-points"); + if (fd == kInvalidFd) return; + WriteToFile(fd, out.data(), out.length()); + CloseFile(fd); + + fd = CovOpenFile(&path, false, "trace-compunits"); + if (fd == kInvalidFd) return; + out.clear(); + for (uptr i = 0; i < comp_unit_name_vec.size(); i++) + out.append("%s\n", comp_unit_name_vec[i].copied_module_name); + WriteToFile(fd, out.data(), out.length()); + CloseFile(fd); + + fd = CovOpenFile(&path, false, "trace-events"); + if (fd == kInvalidFd) return; + uptr bytes_to_write = max_idx * sizeof(tr_event_array[0]); + u8 *event_bytes = reinterpret_cast<u8*>(tr_event_array); + // The trace file could be huge, and may not be written with a single syscall. + while (bytes_to_write) { + uptr actually_written; + if (WriteToFile(fd, event_bytes, bytes_to_write, &actually_written) && + actually_written <= bytes_to_write) { + bytes_to_write -= actually_written; + event_bytes += actually_written; + } else { + break; + } + } + CloseFile(fd); + VReport(1, " CovDump: Trace: %zd PCs written\n", size()); + VReport(1, " CovDump: Trace: %zd Events written\n", max_idx); +} + // This function dumps the caller=>callee pairs into a file as a sequence of // lines like "module_name offset". void CoverageData::DumpCallerCalleePairs() { @@ -347,88 +660,166 @@ void CoverageData::DumpCallerCalleePairs() { callee_module_address); } } - int fd = CovOpenFile(false, "caller-callee"); - if (fd < 0) return; - internal_write(fd, out.data(), out.length()); - internal_close(fd); + InternalScopedString path(kMaxPathLength); + fd_t fd = CovOpenFile(&path, false, "caller-callee"); + if (fd == kInvalidFd) return; + WriteToFile(fd, out.data(), out.length()); + CloseFile(fd); VReport(1, " CovDump: %zd caller-callee pairs written\n", total); } -// Dump the coverage on disk. -static void CovDump() { - if (!common_flags()->coverage || common_flags()->coverage_direct) return; -#if !SANITIZER_WINDOWS - if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed)) - return; - uptr size = coverage_data.size(); - InternalMmapVector<u32> offsets(size); - uptr *vb = coverage_data.data(); - uptr *ve = vb + size; - SortArray(vb, size); - MemoryMappingLayout proc_maps(/*cache_enabled*/true); - uptr mb, me, off, prot; - InternalScopedBuffer<char> module(4096); - InternalScopedBuffer<char> path(4096 * 2); - for (int i = 0; - proc_maps.Next(&mb, &me, &off, module.data(), module.size(), &prot); - i++) { - if ((prot & MemoryMappingLayout::kProtectionExecute) == 0) - continue; - while (vb < ve && *vb < mb) vb++; - if (vb >= ve) break; - if (*vb < me) { - offsets.clear(); - const uptr *old_vb = vb; - CHECK_LE(off, *vb); - for (; vb < ve && *vb < me; vb++) { - uptr diff = *vb - (i ? mb : 0) + off; - CHECK_LE(diff, 0xffffffffU); - offsets.push_back(static_cast<u32>(diff)); - } - const char *module_name = StripModuleName(module.data()); - if (cov_sandboxed) { - if (cov_fd >= 0) { - CovWritePacked(internal_getpid(), module_name, offsets.data(), - offsets.size() * sizeof(u32)); - VReport(1, " CovDump: %zd PCs written to packed file\n", vb - old_vb); - } - } else { - // One file per module per process. - internal_snprintf((char *)path.data(), path.size(), "%s/%s.%zd.sancov", - common_flags()->coverage_dir, module_name, - internal_getpid()); - int fd = CovOpenFile(false /* packed */, module_name); - if (fd > 0) { - internal_write(fd, offsets.data(), offsets.size() * sizeof(u32)); - internal_close(fd); - VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), - vb - old_vb); - } +// Record the current PC into the event buffer. +// Every event is a u32 value (index in tr_pc_array_index) so we compute +// it once and then cache in the provided 'cache' storage. +// +// This function will eventually be inlined by the compiler. +void CoverageData::TraceBasicBlock(s32 *id) { + // Will trap here if + // 1. coverage is not enabled at run-time. + // 2. The array tr_event_array is full. + *tr_event_pointer = static_cast<u32>(*id - 1); + tr_event_pointer++; +} + +void CoverageData::DumpCounters() { + if (!common_flags()->coverage_counters) return; + uptr n = coverage_data.GetNumberOf8bitCounters(); + if (!n) return; + InternalScopedBuffer<u8> bitset(n); + coverage_data.Update8bitCounterBitsetAndClearCounters(bitset.data()); + InternalScopedString path(kMaxPathLength); + + for (uptr m = 0; m < module_name_vec.size(); m++) { + auto r = module_name_vec[m]; + CHECK(r.copied_module_name); + CHECK_LE(r.beg, r.end); + CHECK_LE(r.end, size()); + const char *base_name = StripModuleName(r.copied_module_name); + fd_t fd = + CovOpenFile(&path, /* packed */ false, base_name, "counters-sancov"); + if (fd == kInvalidFd) return; + WriteToFile(fd, bitset.data() + r.beg, r.end - r.beg); + CloseFile(fd); + VReport(1, " CovDump: %zd counters written for '%s'\n", r.end - r.beg, + base_name); + } +} + +void CoverageData::DumpAsBitSet() { + if (!common_flags()->coverage_bitset) return; + if (!size()) return; + InternalScopedBuffer<char> out(size()); + InternalScopedString path(kMaxPathLength); + for (uptr m = 0; m < module_name_vec.size(); m++) { + uptr n_set_bits = 0; + auto r = module_name_vec[m]; + CHECK(r.copied_module_name); + CHECK_LE(r.beg, r.end); + CHECK_LE(r.end, size()); + for (uptr i = r.beg; i < r.end; i++) { + uptr pc = UnbundlePc(pc_array[i]); + out[i] = pc ? '1' : '0'; + if (pc) + n_set_bits++; + } + const char *base_name = StripModuleName(r.copied_module_name); + fd_t fd = CovOpenFile(&path, /* packed */false, base_name, "bitset-sancov"); + if (fd == kInvalidFd) return; + WriteToFile(fd, out.data() + r.beg, r.end - r.beg); + CloseFile(fd); + VReport(1, + " CovDump: bitset of %zd bits written for '%s', %zd bits are set\n", + r.end - r.beg, base_name, n_set_bits); + } +} + +void CoverageData::DumpOffsets() { + auto sym = Symbolizer::GetOrInit(); + if (!common_flags()->coverage_pcs) return; + CHECK_NE(sym, nullptr); + InternalMmapVector<uptr> offsets(0); + InternalScopedString path(kMaxPathLength); + for (uptr m = 0; m < module_name_vec.size(); m++) { + offsets.clear(); + uptr num_words_for_magic = SANITIZER_WORDSIZE == 64 ? 1 : 2; + for (uptr i = 0; i < num_words_for_magic; i++) + offsets.push_back(0); + auto r = module_name_vec[m]; + CHECK(r.copied_module_name); + CHECK_LE(r.beg, r.end); + CHECK_LE(r.end, size()); + for (uptr i = r.beg; i < r.end; i++) { + uptr pc = UnbundlePc(pc_array[i]); + uptr counter = UnbundleCounter(pc_array[i]); + if (!pc) continue; // Not visited. + uptr offset = 0; + sym->GetModuleNameAndOffsetForPC(pc, nullptr, &offset); + offsets.push_back(BundlePcAndCounter(offset, counter)); + } + + CHECK_GE(offsets.size(), num_words_for_magic); + SortArray(offsets.data(), offsets.size()); + for (uptr i = 0; i < offsets.size(); i++) + offsets[i] = UnbundlePc(offsets[i]); + + uptr num_offsets = offsets.size() - num_words_for_magic; + u64 *magic_p = reinterpret_cast<u64*>(offsets.data()); + CHECK_EQ(*magic_p, 0ULL); + // FIXME: we may want to write 32-bit offsets even in 64-mode + // if all the offsets are small enough. + *magic_p = SANITIZER_WORDSIZE == 64 ? kMagic64 : kMagic32; + + const char *module_name = StripModuleName(r.copied_module_name); + if (cov_sandboxed) { + if (cov_fd != kInvalidFd) { + CovWritePacked(internal_getpid(), module_name, offsets.data(), + offsets.size() * sizeof(offsets[0])); + VReport(1, " CovDump: %zd PCs written to packed file\n", num_offsets); } + } else { + // One file per module per process. + fd_t fd = CovOpenFile(&path, false /* packed */, module_name); + if (fd == kInvalidFd) continue; + WriteToFile(fd, offsets.data(), offsets.size() * sizeof(offsets[0])); + CloseFile(fd); + VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), num_offsets); } } - if (cov_fd >= 0) - internal_close(cov_fd); - coverage_data.DumpCallerCalleePairs(); -#endif // !SANITIZER_WINDOWS + if (cov_fd != kInvalidFd) + CloseFile(cov_fd); +} + +void CoverageData::DumpAll() { + if (!coverage_enabled || common_flags()->coverage_direct) return; + if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed)) + return; + DumpAsBitSet(); + DumpCounters(); + DumpTrace(); + DumpOffsets(); + DumpCallerCalleePairs(); } void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { if (!args) return; - if (!common_flags()->coverage) return; + if (!coverage_enabled) return; cov_sandboxed = args->coverage_sandboxed; if (!cov_sandboxed) return; - cov_fd = args->coverage_fd; cov_max_block_size = args->coverage_max_block_size; - if (cov_fd < 0) + if (args->coverage_fd >= 0) { + cov_fd = (fd_t)args->coverage_fd; + } else { + InternalScopedString path(kMaxPathLength); // Pre-open the file now. The sandbox won't allow us to do it later. - cov_fd = CovOpenFile(true /* packed */, 0); + cov_fd = CovOpenFile(&path, true /* packed */, nullptr); + } } -int MaybeOpenCovFile(const char *name) { +fd_t MaybeOpenCovFile(const char *name) { CHECK(name); - if (!common_flags()->coverage) return -1; - return CovOpenFile(true /* packed */, name); + if (!coverage_enabled) return kInvalidFd; + InternalScopedString path(kMaxPathLength); + return CovOpenFile(&path, true /* packed */, name); } void CovBeforeFork() { @@ -439,32 +830,114 @@ void CovAfterFork(int child_pid) { coverage_data.AfterFork(child_pid); } -} // namespace __sanitizer +static void MaybeDumpCoverage() { + if (common_flags()->coverage) + __sanitizer_cov_dump(); +} + +void InitializeCoverage(bool enabled, const char *dir) { + if (coverage_enabled) + return; // May happen if two sanitizer enable coverage in the same process. + coverage_enabled = enabled; + coverage_dir = dir; + coverage_data.Init(); + if (enabled) coverage_data.Enable(); + if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump); + AddDieCallback(MaybeDumpCoverage); +} + +void ReInitializeCoverage(bool enabled, const char *dir) { + coverage_enabled = enabled; + coverage_dir = dir; + coverage_data.ReInit(); +} + +void CoverageUpdateMapping() { + if (coverage_enabled) + CovUpdateMapping(coverage_dir); +} + +} // namespace __sanitizer extern "C" { -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov() { - coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC())); +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) { + coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()), + guard); +} +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_with_check(u32 *guard) { + atomic_uint32_t *atomic_guard = reinterpret_cast<atomic_uint32_t*>(guard); + if (static_cast<s32>( + __sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) < 0) + __sanitizer_cov(guard); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_indir_call16(uptr callee, uptr callee_cache16[]) { coverage_data.IndirCall(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()), callee, callee_cache16, 16); } -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { CovDump(); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() { + coverage_enabled = true; + coverage_dir = common_flags()->coverage_dir; coverage_data.Init(); } -SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(uptr npcs) { - if (!common_flags()->coverage || !common_flags()->coverage_direct) return; - if (SANITIZER_ANDROID) { +SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { + coverage_data.DumpAll(); +} +SANITIZER_INTERFACE_ATTRIBUTE void +__sanitizer_cov_module_init(s32 *guards, uptr npcs, u8 *counters, + const char *comp_unit_name) { + coverage_data.InitializeGuards(guards, npcs, comp_unit_name, GET_CALLER_PC()); + coverage_data.InitializeCounters(counters, npcs); + if (!common_flags()->coverage_direct) return; + if (SANITIZER_ANDROID && coverage_enabled) { // dlopen/dlclose interceptors do not work on Android, so we rely on // Extend() calls to update .sancov.map. - CovUpdateMapping(GET_CALLER_PC()); + CovUpdateMapping(coverage_dir, GET_CALLER_PC()); } coverage_data.Extend(npcs); } SANITIZER_INTERFACE_ATTRIBUTE sptr __sanitizer_maybe_open_cov_file(const char *name) { - return MaybeOpenCovFile(name); + return (sptr)MaybeOpenCovFile(name); +} +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_get_total_unique_coverage() { + return atomic_load(&coverage_counter, memory_order_relaxed); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_cov_trace_func_enter(s32 *id) { + coverage_data.TraceBasicBlock(id); +} +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_cov_trace_basic_block(s32 *id) { + coverage_data.TraceBasicBlock(id); +} +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_reset_coverage() { + coverage_data.ReinitializeGuards(); + internal_bzero_aligned16( + coverage_data.data(), + RoundUpTo(coverage_data.size() * sizeof(coverage_data.data()[0]), 16)); +} +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_get_coverage_guards(uptr **data) { + *data = coverage_data.data(); + return coverage_data.size(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_get_number_of_counters() { + return coverage_data.GetNumberOf8bitCounters(); +} + +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_update_counter_bitset_and_clear_counters(u8 *bitset) { + return coverage_data.Update8bitCounterBitsetAndClearCounters(bitset); } -} // extern "C" +// Default empty implementations (weak). Users should redefine them. +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_cmp() {} +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_switch() {} +} // extern "C" |