rts: Assert that pointers aren't cleared by -DZ

This turns many segmentation faults into much easier-to-debug assertion
failures by ensuring that LOOKS_LIKE_*_PTR checks recognize bit-patterns
produced by `+RTS -DZ` clearing as invalid pointers.

This is a bit ad-hoc but this is the debug runtime.

3 files changed, 51 insertions, 18 deletions

diff --git a/rts/include/Cmm.h b/rts/include/Cmm.h
index a1cf44c31b..7936bc8b85 100644
--- a/rts/include/Cmm.h
+++ b/rts/include/Cmm.h
@@ -607,16 +607,20 @@
 #define BITMAP_SIZE(bitmap) ((bitmap) & BITMAP_SIZE_MASK)
 #define BITMAP_BITS(bitmap) ((bitmap) >> BITMAP_BITS_SHIFT)
 
+#define LOOKS_LIKE_PTR(p) ((p) != NULL && (p) != INVALID_GHC_POINTER)
+
 /* Debugging macros */
 #define LOOKS_LIKE_INFO_PTR(p)                                  \
-   ((p) != NULL &&                                              \
+   (LOOKS_LIKE_PTR(p) &&                                        \
     LOOKS_LIKE_INFO_PTR_NOT_NULL(p))
 
 #define LOOKS_LIKE_INFO_PTR_NOT_NULL(p)                         \
    ( (TO_W_(%INFO_TYPE(%STD_INFO(p))) != INVALID_OBJECT) &&     \
      (TO_W_(%INFO_TYPE(%STD_INFO(p))) <  N_CLOSURE_TYPES))
 
-#define LOOKS_LIKE_CLOSURE_PTR(p) (LOOKS_LIKE_INFO_PTR(GET_INFO(UNTAG(p))))
+#define LOOKS_LIKE_CLOSURE_PTR(p)                               \
+   ( LOOKS_LIKE_PTR(p) &&                                       \
+     LOOKS_LIKE_INFO_PTR(GET_INFO(UNTAG(p))))
 
 /*
  * The layout of the StgFunInfoExtra part of an info table changes
diff --git a/rts/include/rts/Constants.h b/rts/include/rts/Constants.h
index 3bf5a7a2d5..e18a2c7bb9 100644
--- a/rts/include/rts/Constants.h
+++ b/rts/include/rts/Constants.h
@@ -215,6 +215,21 @@
 #define LDV_STATE_USE           0x40000000
 #endif /* SIZEOF_VOID_P */
 
+/* See Note [Debugging predicates for pointers] in ClosureMacros.h */
+#if !defined(INVALID_GHC_POINTER)
+#if !defined(DEBUG)
+#define INVALID_GHC_POINTER 0x0
+#elif SIZEOF_VOID_P== 4
+/* N.B. this may result in false-negatives from LOOKS_LIKE_PTR on some
+ * platforms since this is a valid user-space address.
+ */
+#define INVALID_GHC_POINTER 0xaaaaaaaa
+#else
+/* N.B. this is typically a kernel-mode address on 64-bit platforms */
+#define INVALID_GHC_POINTER 0xaaaaaaaaaaaaaaaa
+#endif
+#endif
+
 /* -----------------------------------------------------------------------------
    TSO related constants
    -------------------------------------------------------------------------- */
diff --git a/rts/include/rts/storage/ClosureMacros.h b/rts/include/rts/storage/ClosureMacros.h
index 57e9be2de8..4a5d6dd3d8 100644
--- a/rts/include/rts/storage/ClosureMacros.h
+++ b/rts/include/rts/storage/ClosureMacros.h
@@ -253,22 +253,35 @@ EXTERN_INLINE StgClosure *TAG_CLOSURE(StgWord tag,StgClosure * p)
 #define MK_FORWARDING_PTR(p) (((StgWord)p) | 1)
 #define UN_FORWARDING_PTR(p) (((StgWord)p) - 1)
 
-/* -----------------------------------------------------------------------------
-   DEBUGGING predicates for pointers
-
-   LOOKS_LIKE_INFO_PTR(p)    returns False if p is definitely not an info ptr
-   LOOKS_LIKE_CLOSURE_PTR(p) returns False if p is definitely not a closure ptr
-
-   These macros are complete but not sound.  That is, they might
-   return false positives.  Do not rely on them to distinguish info
-   pointers from closure pointers, for example.
+/*
+ * Note [Debugging predicates for pointers]
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ * LOOKS_LIKE_PTR(p)         returns False if p is definitely not a valid pointer
+ * LOOKS_LIKE_INFO_PTR(p)    returns False if p is definitely not an info ptr
+ * LOOKS_LIKE_CLOSURE_PTR(p) returns False if p is definitely not a closure ptr
+ *
+ * These macros are complete but not sound.  That is, they might
+ * return false positives.  Do not rely on them to distinguish info
+ * pointers from closure pointers, for example.
+ *
+ * We for the most part don't use address-space predicates these days, for
+ * portability reasons, and the fact that code/data can be scattered about the
+ * address space in a dynamically-linked environment.  Our best option is to
+ * look at the alleged info table and see whether it seems to make sense.
+ *
+ * The one exception here is the use of INVALID_GHC_POINTER, which catches
+ * the bit-pattern used by `+RTS -DZ` to zero freed memory (that is 0xaaaaa...).
+ * In the case of most 64-bit platforms, this INVALID_GHC_POINTER is a
+ * kernel-mode address, making this check free of false-negatives. On the other
+ * hand, on 32-bit platforms this typically isn't the case. Consequently, we
+ * only use this check in the DEBUG RTS.
+ */
 
-   We don't use address-space predicates these days, for portability
-   reasons, and the fact that code/data can be scattered about the
-   address space in a dynamically-linked environment.  Our best option
-   is to look at the alleged info table and see whether it seems to
-   make sense...
-   -------------------------------------------------------------------------- */
+EXTERN_INLINE bool LOOKS_LIKE_PTR (const void* p);
+EXTERN_INLINE bool LOOKS_LIKE_PTR (const void* p)
+{
+    return p && (p != (const void*) INVALID_GHC_POINTER);
+}
 
 EXTERN_INLINE bool LOOKS_LIKE_INFO_PTR_NOT_NULL (StgWord p);
 EXTERN_INLINE bool LOOKS_LIKE_INFO_PTR_NOT_NULL (StgWord p)
@@ -280,12 +293,13 @@ EXTERN_INLINE bool LOOKS_LIKE_INFO_PTR_NOT_NULL (StgWord p)
 EXTERN_INLINE bool LOOKS_LIKE_INFO_PTR (StgWord p);
 EXTERN_INLINE bool LOOKS_LIKE_INFO_PTR (StgWord p)
 {
-    return p && (IS_FORWARDING_PTR(p) || LOOKS_LIKE_INFO_PTR_NOT_NULL(p));
+    return LOOKS_LIKE_PTR((const void*) p) && (IS_FORWARDING_PTR(p) || LOOKS_LIKE_INFO_PTR_NOT_NULL(p));
 }
 
 EXTERN_INLINE bool LOOKS_LIKE_CLOSURE_PTR (const void *p);
 EXTERN_INLINE bool LOOKS_LIKE_CLOSURE_PTR (const void *p)
 {
+    if (!LOOKS_LIKE_PTR(p)) return false;
     const StgInfoTable *info = RELAXED_LOAD(&UNTAG_CONST_CLOSURE((const StgClosure *) (p))->header.info);
     return LOOKS_LIKE_INFO_PTR((StgWord) info);
 }