[libc] add exception to atof differential fuzz

The differential fuzzer for atof found a bug in glibc's handling of
hexadecimal rounding. Since we can't easily update glibc and we want to
avoid false positives when running the fuzzer, I've added an exception
to skip all hexadecimal subnormal cases.

Reviewed By: sivachandra

Differential Revision: https://reviews.llvm.org/D149359

2 files changed, 40 insertions, 1 deletions

diff --git a/libc/fuzzing/stdlib/CMakeLists.txt b/libc/fuzzing/stdlib/CMakeLists.txt
index 3127848ee193..785efd82a1ed 100644
--- a/libc/fuzzing/stdlib/CMakeLists.txt
+++ b/libc/fuzzing/stdlib/CMakeLists.txt
@@ -14,6 +14,8 @@ add_libc_fuzzer(
     StringParserOutputDiff.h
   DEPENDS
     libc.src.stdlib.atof
+  COMPILE_OPTIONS
+    -DLLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
 )
 
 add_libc_fuzzer(
diff --git a/libc/fuzzing/stdlib/atof_differential_fuzz.cpp b/libc/fuzzing/stdlib/atof_differential_fuzz.cpp
index b368129960d3..2f330e1af2e3 100644
--- a/libc/fuzzing/stdlib/atof_differential_fuzz.cpp
+++ b/libc/fuzzing/stdlib/atof_differential_fuzz.cpp
@@ -16,6 +16,40 @@
 
 #include "fuzzing/stdlib/StringParserOutputDiff.h"
 
+// TODO: Remove this once glibc fixes hex subnormal rounding. See
+// https://sourceware.org/bugzilla/show_bug.cgi?id=30220
+#ifdef LLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
+#include <ctype.h>
+constexpr double MIN_NORMAL = 0x1p-1022;
+
+bool has_hex_prefix(const uint8_t *str) {
+  size_t index = 0;
+
+  // Skip over leading whitespace
+  while (isspace(str[index])) {
+    ++index;
+  }
+  // Skip over sign
+  if (str[index] == '-' || str[index] == '+') {
+    ++index;
+  }
+  return str[index] == '0' && (tolower(str[index + 1])) == 'x';
+}
+
+bool should_be_skipped(const uint8_t *str) {
+  double init_result = ::atof(reinterpret_cast<const char *>(str));
+  if (init_result < 0) {
+    init_result = -init_result;
+  }
+  if (init_result < MIN_NORMAL && init_result != 0) {
+    return has_hex_prefix(str);
+  }
+  return false;
+}
+#else
+bool should_be_skipped(const uint8_t *) { return false; }
+#endif // LLVM_LIBC_ATOF_DIF_FUZZ_SKIP_GLIBC_HEX_SUBNORMAL_ERR
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   uint8_t *container = new uint8_t[size + 1];
   if (!container)
@@ -26,7 +60,10 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     container[i] = data[i];
   container[size] = '\0'; // Add null terminator to container.
 
-  StringParserOutputDiff<double>(&__llvm_libc::atof, &::atof, container, size);
+  if (!should_be_skipped(container)) {
+    StringParserOutputDiff<double>(&__llvm_libc::atof, &::atof, container,
+                                   size);
+  }
   delete[] container;
   return 0;
 }