rts/posix: Reduce heap allocation amount on mmap failure

Since the two-step allocator the RTS asks the kernel for a large upfront
mmap'd region of memory (on the order of terabytes). While we have no
expectation that this entire region will be backed by physical memory,
this scheme nevertheless fails on some systems with resource limits.
Here we use a back-off scheme to reduce our allocation request until we
find a size agreeable to the kernel. Fixes #10877.

This also fixes a latent bug wherein the heap reservation retry logic
would fail to free the previously reserved address space, which would
likely result in a heap allocation failure.

Test Plan:
set address space limit with `ulimit -v 67108864` and try running
a compiled program

Reviewers: simonmar, austin

Reviewed By: simonmar

Subscribers: thomie, RyanGlScott

Differential Revision: https://phabricator.haskell.org/D1405

GHC Trac Issues: #10877

4 files changed, 61 insertions, 12 deletions

diff --git a/rts/posix/OSMem.c b/rts/posix/OSMem.c
index 43c7831a37..274d5ada73 100644
--- a/rts/posix/OSMem.c
+++ b/rts/posix/OSMem.c
@@ -102,6 +102,7 @@ enum
     MEM_RESERVE_AND_COMMIT = MEM_RESERVE | MEM_COMMIT
 };
 
+/* Returns NULL on failure; errno set */
 static void *
 my_mmap (void *addr, W_ size, int operation)
 {
@@ -196,6 +197,19 @@ my_mmap (void *addr, W_ size, int operation)
 #endif
 
     if (ret == (void *)-1) {
+        return NULL;
+    }
+
+    return ret;
+}
+
+/* Variant of my_mmap which aborts in the case of an error */
+static void *
+my_mmap_or_barf (void *addr, W_ size, int operation)
+{
+    void *ret = my_mmap(addr, size, operation);
+
+    if (ret == NULL) {
         if (errno == ENOMEM ||
             (errno == EINVAL && sizeof(void*)==4 && size >= 0xc0000000)) {
             // If we request more than 3Gig, then we get EINVAL
@@ -222,7 +236,7 @@ gen_map_mblocks (W_ size)
     // Try to map a larger block, and take the aligned portion from
     // it (unmap the rest).
     size += MBLOCK_SIZE;
-    ret = my_mmap(0, size, MEM_RESERVE_AND_COMMIT);
+    ret = my_mmap_or_barf(0, size, MEM_RESERVE_AND_COMMIT);
 
     // unmap the slop bits around the chunk we allocated
     slop = (W_)ret & MBLOCK_MASK;
@@ -261,7 +275,7 @@ osGetMBlocks(nat n)
       // use gen_map_mblocks the first time.
       ret = gen_map_mblocks(size);
   } else {
-      ret = my_mmap(next_request, size, MEM_RESERVE_AND_COMMIT);
+      ret = my_mmap_or_barf(next_request, size, MEM_RESERVE_AND_COMMIT);
 
       if (((W_)ret & MBLOCK_MASK) != 0) {
           // misaligned block!
@@ -387,6 +401,9 @@ osTryReserveHeapMemory (W_ len, void *hint)
        and then we discard what we don't need */
 
     base = my_mmap(hint, len + MBLOCK_SIZE, MEM_RESERVE);
+    if (base == NULL)
+        return NULL;
+
     top = (void*)((W_)base + len + MBLOCK_SIZE);
 
     if (((W_)base & MBLOCK_MASK) != 0) {
@@ -407,7 +424,7 @@ osTryReserveHeapMemory (W_ len, void *hint)
     return start;
 }
 
-void *osReserveHeapMemory(W_ len)
+void *osReserveHeapMemory(W_ *len)
 {
     int attempt;
     void *at;
@@ -419,15 +436,43 @@ void *osReserveHeapMemory(W_ len)
        libraries are position independent but cannot be loaded about 4GB.
 
        We do so with a hint to the mmap, and we verify the OS satisfied our
-       hint. We loop a few times in case there is already something allocated
-       there, but we bail if we cannot allocate at all.
+       hint. We loop, shifting our hint by 1 BLOCK_SIZE every time, in case
+       there is already something allocated there.
+
+       Some systems impose resource limits restricting the amount of memory we
+       can request (see, e.g. #10877). If mmap fails we halve our allocation
+       request and try again. If our request size gets absurdly small we simply
+       give up.
+
     */
 
     attempt = 0;
-    do {
+    while (1) {
+        if (*len < MBLOCK_SIZE) {
+            // Give up if the system won't even give us 16 blocks worth of heap
+            barf("osReserveHeapMemory: Failed to allocate heap storage");
+        }
+
         void *hint = (void*)((W_)8 * (1 << 30) + attempt * BLOCK_SIZE);
-        at = osTryReserveHeapMemory(len, hint);
-    } while ((W_)at < ((W_)8 * (1 << 30)));
+        at = osTryReserveHeapMemory(*len, hint);
+        if (at == NULL) {
+            // This means that mmap failed which we take to mean that we asked
+            // for too much memory. This can happen due to POSIX resource
+            // limits. In this case we reduce our allocation request by a factor
+            // of two and try again.
+            *len /= 2;
+        } else if ((W_)at >= ((W_)8 * (1 << 30))) {
+            // Success! We were given a block of memory starting above the 8 GB
+            // mark, which is what we were looking for.
+            break;
+        } else {
+            // We got addressing space but it wasn't above the 8GB mark.
+            // Try again.
+            if (munmap(at, *len) < 0) {
+                sysErrorBelch("unable to release reserved heap");
+            }
+        }
+    }
 
     return at;
 }
diff --git a/rts/sm/MBlock.c b/rts/sm/MBlock.c
index e1daa71e2f..2131ae6e13 100644
--- a/rts/sm/MBlock.c
+++ b/rts/sm/MBlock.c
@@ -645,7 +645,7 @@ initMBlocks(void)
 #else
         size = (W_)1 << 40; // 1 TByte
 #endif
-        void *addr = osReserveHeapMemory(size);
+        void *addr = osReserveHeapMemory(&size);
 
         mblock_address_space.begin = (W_)addr;
         mblock_address_space.end = (W_)addr + size;
diff --git a/rts/sm/OSMem.h b/rts/sm/OSMem.h
index 6bcaf65b10..533f6f7fe6 100644
--- a/rts/sm/OSMem.h
+++ b/rts/sm/OSMem.h
@@ -34,8 +34,12 @@ void setExecutable (void *p, W_ len, rtsBool exec);
 // pointed to by the return value, until that memory is committed using
 // osCommitMemory().
 //
+// The value pointed to by len will be filled by the caller with an upper
+// bound on the amount of memory to reserve. On return this will be set
+// to the amount of memory actually reserved.
+//
 // This function is called once when the block allocator is initialized.
-void *osReserveHeapMemory(W_ len);
+void *osReserveHeapMemory(W_ *len);
 
 // Commit (allocate memory for) a piece of address space, which must
 // be within the previously reserved space After this call, it is safe
diff --git a/rts/win32/OSMem.c b/rts/win32/OSMem.c
index 2d2af0ddf6..47e24f077a 100644
--- a/rts/win32/OSMem.c
+++ b/rts/win32/OSMem.c
@@ -429,11 +429,11 @@ void setExecutable (void *p, W_ len, rtsBool exec)
 
 static void* heap_base = NULL;
 
-void *osReserveHeapMemory (W_ len)
+void *osReserveHeapMemory (W_ *len)
 {
     void *start;
 
-    heap_base = VirtualAlloc(NULL, len + MBLOCK_SIZE,
+    heap_base = VirtualAlloc(NULL, *len + MBLOCK_SIZE,
                               MEM_RESERVE, PAGE_READWRITE);
     if (heap_base == NULL) {
         if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY) {