Support GC_memalign with alignments greater than HBLKSIZE

Issue #510 (bdwgc).

* allchblk.c (GC_allochblk_nth, GC_allochblk, next_hblk_fits_better,
find_nonbl_hblk): Add align_m1 argument.
* include/private/gc_priv.h (GC_allochblk, GC_alloc_large): Likewise.
* malloc.c (GC_alloc_large): Likewise.
* allchblk.c (GC_allochblk): Check that there is no overflow in
blocks*HBLKSIZE+align_m1; pass align_m1 to GC_allochblk_nth; try
start_list again (with may_split) if non-zero align_m1.
* allchblk.c (ALIGN_PAD_SZ): New macro.
* allchblk.c (next_hblk_fits_better): Define and use next_ofs local
variable; adjust next_hbp passed to GC_is_black_listed().
* allchblk.c (find_nonbl_hblk): Adjust search_end and last_hbp based
on align_m1 value.
* allchblk.c (GC_allochblk_nth): Add assertion that align_m1+1 is
a power of two and that align_m1+1 is 1 or a multiple of HBLKSIZE;
define and use align_ofs local variable; add assertion that last_hbp
(before GC_install_header() call) is multiple of align_m1+1.
* include/gc/gc.h (GC_memalign): Update comment to mention the
restriction on align value (should be a power of two and not less than
size of a pointer).
* include/private/gc_priv.h (GC_allochblk, GC_alloc_large): Update
comment (to mention align_m1 argument).
* include/private/gc_priv.h (GC_generic_malloc_aligned): Declare
function.
* include/private/gcconfig.h [(NACL || I386 && (EMSCRIPTEN || WASI))
&& !HBLKSIZE && !GC_NO_VALLOC] (HBLKSIZE): Do not define (to 65536 or
GETPAGESIZE()); remove TODO item.
* malloc.c (GC_alloc_large): Add alignment to n_blocks value (but not
to GC_large_allocd_bytes); pass align_m1 to GC_allochblk(); add
assertion that result is aligned to align_m1+1.
* malloc.c (GC_alloc_large_and_clear): Pass zero align_m1 to
GC_alloc_large().
* mallocx.c (GC_generic_malloc_ignore_off_page): Likewise.
* malloc.c (GC_generic_malloc_aligned): Move code from
GC_generic_malloc; adjust align_m1 to be either zero or not less than
HBLKSIZE-1, and pass it to GC_alloc_large(); add comment that the
result of GC_oom_fn(lb) might be unaligned.
* malloc.c (GC_generic_malloc): Call GC_generic_malloc_aligned() with
zero align_m1.
* mallocx.c (GC_generic_malloc_ignore_off_page): Expect result of
GC_alloc_large() is unlikely to be NULL.
* mallocx.c (GC_generic_malloc_many): Pass zero align_m1 to
GC_allochblk().
* new_hblk.c (GC_new_hblk): Likewise.
* mallocx.c: Do not include limits.h.
* mallocx.c (GC_memalign): Remove new_lb local variable; define and
use align_m1 local variable; if align is smaller than sizeof(void*) or
is not a power of two than return NULL; call GC_generic_malloc_aligned
and GC_clear_stack (instead of GC_oom_fn() or GC_malloc()) if align or
lb is at least HBLKSIZE/2; replace result%align to result&align_m1;
do not result+=offset if offset is zero.
* new_hblk.c (GC_new_hblk): Expect result of GC_allochblk() is unlikely
to be NULL.
* tests/gctest.c (run_one_test): Call GC_memalign() also for bigger
alignment values (up to HBLKSIZE*4).

1 files changed, 26 insertions, 13 deletions

diff --git a/malloc.c b/malloc.c
index e0643790..5101df64 100644
--- a/malloc.c
+++ b/malloc.c
@@ -33,16 +33,17 @@ STATIC GC_bool GC_alloc_reclaim_list(struct obj_kind *kind)
     return TRUE;
 }
 
-GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags)
+GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags,
+                              size_t align_m1)
 {
     struct hblk * h;
-    size_t n_blocks;
+    size_t n_blocks; /* includes alignment */
     ptr_t result = NULL;
     GC_bool retry = FALSE;
 
     GC_ASSERT(I_HOLD_LOCK());
     lb = ROUNDUP_GRANULE_SIZE(lb);
-    n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(lb);
+    n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(SIZET_SAT_ADD(lb, align_m1));
     if (!EXPECT(GC_is_initialized, TRUE)) {
       DCL_LOCK_STATE;
       UNLOCK(); /* just to unset GC_lock_holder */
@@ -56,25 +57,26 @@ GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags)
             EXIT_GC();
     }
 
-    h = GC_allochblk(lb, k, flags);
+    h = GC_allochblk(lb, k, flags, align_m1);
 #   ifdef USE_MUNMAP
         if (NULL == h) {
             GC_merge_unmapped();
-            h = GC_allochblk(lb, k, flags);
+            h = GC_allochblk(lb, k, flags, align_m1);
         }
 #   endif
     while (0 == h && GC_collect_or_expand(n_blocks, flags != 0, retry)) {
-        h = GC_allochblk(lb, k, flags);
+        h = GC_allochblk(lb, k, flags, align_m1);
         retry = TRUE;
     }
     if (EXPECT(h != NULL, TRUE)) {
-        if (n_blocks > 1) {
-            GC_large_allocd_bytes += HBLKSIZE * n_blocks;
+        if (lb > HBLKSIZE) {
+            GC_large_allocd_bytes += HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb);
             if (GC_large_allocd_bytes > GC_max_large_allocd_bytes)
                 GC_max_large_allocd_bytes = GC_large_allocd_bytes;
         }
         /* FIXME: Do we need some way to reset GC_max_large_allocd_bytes? */
         result = h -> hb_body;
+        GC_ASSERT(((word)result & align_m1) == 0);
     }
     return result;
 }
@@ -86,7 +88,7 @@ STATIC ptr_t GC_alloc_large_and_clear(size_t lb, int k, unsigned flags)
     ptr_t result;
 
     GC_ASSERT(I_HOLD_LOCK());
-    result = GC_alloc_large(lb, k, flags);
+    result = GC_alloc_large(lb, k, flags, 0 /* align_m1 */);
     if (EXPECT(result != NULL, TRUE)
           && (GC_debugging_started || GC_obj_kinds[k].ok_init)) {
         /* Clear the whole block, in case of GC_realloc call. */
@@ -234,7 +236,7 @@ GC_INNER void * GC_generic_malloc_inner(size_t lb, int k)
 # endif
 #endif
 
-GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc(size_t lb, int k)
+GC_INNER void * GC_generic_malloc_aligned(size_t lb, int k, size_t align_m1)
 {
     void * result;
     DCL_LOCK_STATE;
@@ -244,7 +246,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc(size_t lb, int k)
       GC_print_all_errors();
     GC_INVOKE_FINALIZERS();
     GC_DBG_COLLECT_AT_MALLOC(lb);
-    if (SMALL_OBJ(lb)) {
+    if (SMALL_OBJ(lb) && EXPECT(align_m1 < GRANULE_BYTES, TRUE)) {
         LOCK();
         result = GC_generic_malloc_inner(lb, k);
         UNLOCK();
@@ -256,8 +258,13 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc(size_t lb, int k)
         lg = ROUNDED_UP_GRANULES(lb);
         lb_rounded = GRANULES_TO_BYTES(lg);
         init = GC_obj_kinds[k].ok_init;
+        if (EXPECT(align_m1 < GRANULE_BYTES, TRUE)) {
+          align_m1 = 0;
+        } else if (align_m1 < HBLKSIZE) {
+          align_m1 = HBLKSIZE - 1;
+        }
         LOCK();
-        result = (ptr_t)GC_alloc_large(lb_rounded, k, 0);
+        result = (ptr_t)GC_alloc_large(lb_rounded, k, 0 /* flags */, align_m1);
         if (EXPECT(result != NULL, TRUE)) {
           if (GC_debugging_started) {
             BZERO(result, HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_rounded));
@@ -278,10 +285,16 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc(size_t lb, int k)
             BZERO(result, HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_rounded));
         }
     }
-    if (EXPECT(NULL == result, FALSE)) return (*GC_get_oom_fn())(lb);
+    if (EXPECT(NULL == result, FALSE))
+      result = (*GC_get_oom_fn())(lb); /* might be misaligned */
     return result;
 }
 
+GC_API GC_ATTR_MALLOC void * GC_CALL GC_generic_malloc(size_t lb, int k)
+{
+    return GC_generic_malloc_aligned(lb, k, 0 /* align_m1 */);
+}
+
 GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind_global(size_t lb, int k)
 {
     GC_ASSERT(k < MAXOBJKINDS);