Make integer overflow less likely to happen (#7762)

The particular problematic code in #7762 was this:

            nat newSize = size - n;
            char *freeAddr = MBLOCK_ROUND_DOWN(bd->start);
            freeAddr += newSize * MBLOCK_SIZE;
                        ^^^^^^^^^^^^^^^^^^^^^^  OVERFLOW!!!

For good measure, I'm going to fix the bug twice.  This patch fixes
the class of bugs of this kind, by making sure that any expressions
involving BLOCK_SIZE or MBLOCK_SIZE are promoted to unsigned long.  In
a separate patch, I'll fix a bunch of individual instances (including
the one above).

1 files changed, 24 insertions, 0 deletions

diff --git a/includes/rts/storage/Block.h b/includes/rts/storage/Block.h
index 008602af3d..7b3bc991d3 100644
--- a/includes/rts/storage/Block.h
+++ b/includes/rts/storage/Block.h
@@ -15,7 +15,13 @@
 
 /* Block related constants (BLOCK_SHIFT is defined in Constants.h) */
 
+#ifdef CMINUSMINUS
 #define BLOCK_SIZE   (1<<BLOCK_SHIFT)
+#else
+#define BLOCK_SIZE   (1UL<<BLOCK_SHIFT)
+// Note [integer overflow]
+#endif
+
 #define BLOCK_SIZE_W (BLOCK_SIZE/sizeof(W_))
 #define BLOCK_MASK   (BLOCK_SIZE-1)
 
@@ -24,7 +30,13 @@
 
 /* Megablock related constants (MBLOCK_SHIFT is defined in Constants.h) */
 
+#ifdef CMINUSMINUS
 #define MBLOCK_SIZE    (1<<MBLOCK_SHIFT)
+#else
+#define MBLOCK_SIZE    (1UL<<MBLOCK_SHIFT)
+// Note [integer overflow]
+#endif
+
 #define MBLOCK_SIZE_W  (MBLOCK_SIZE/sizeof(W_))
 #define MBLOCK_MASK    (MBLOCK_SIZE-1)
 
@@ -37,6 +49,18 @@
  */
 #define LARGE_OBJECT_THRESHOLD ((nat)(BLOCK_SIZE * 8 / 10))
 
+/*
+ * Note [integer overflow]
+ *
+ * The UL suffix in BLOCK_SIZE and MBLOCK_SIZE promotes the expression
+ * to an unsigned long, which means that expressions involving these
+ * will be promoted to unsigned long, which makes integer overflow
+ * less likely.  Historically, integer overflow in expressions like
+ *    (n * BLOCK_SIZE)
+ * where n is int or unsigned int, have caused obscure segfaults in
+ * programs that use large amounts of memory (e.g. #7762, #5086).
+ */
+
 /* -----------------------------------------------------------------------------
  * Block descriptor.  This structure *must* be the right length, so we
  * can do pointer arithmetic on pointers to it.