diff options
Diffstat (limited to 'deps/jemalloc/src/large.c')
-rw-r--r-- | deps/jemalloc/src/large.c | 299 |
1 files changed, 113 insertions, 186 deletions
diff --git a/deps/jemalloc/src/large.c b/deps/jemalloc/src/large.c index 8e7a781d3..5fc4bf584 100644 --- a/deps/jemalloc/src/large.c +++ b/deps/jemalloc/src/large.c @@ -1,11 +1,11 @@ -#define JEMALLOC_LARGE_C_ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/assert.h" +#include "jemalloc/internal/emap.h" #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/prof_recent.h" #include "jemalloc/internal/util.h" /******************************************************************************/ @@ -21,8 +21,7 @@ void * large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero) { size_t ausize; - extent_t *extent; - bool is_zeroed; + edata_t *edata; UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false); assert(!tsdn_null(tsdn) || arena != NULL); @@ -32,163 +31,80 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, return NULL; } - if (config_fill && unlikely(opt_zero)) { - zero = true; - } - /* - * Copy zero into is_zeroed and pass the copy when allocating the - * extent, so that it is possible to make correct junk/zero fill - * decisions below, even if is_zeroed ends up true when zero is false. - */ - is_zeroed = zero; if (likely(!tsdn_null(tsdn))) { arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); } - if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn, - arena, usize, alignment, &is_zeroed)) == NULL) { + if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn, + arena, usize, alignment, zero)) == NULL) { return NULL; } /* See comments in arena_bin_slabs_full_insert(). */ if (!arena_is_auto(arena)) { - /* Insert extent into large. */ + /* Insert edata into large. */ malloc_mutex_lock(tsdn, &arena->large_mtx); - extent_list_append(&arena->large, extent); + edata_list_active_append(&arena->large, edata); malloc_mutex_unlock(tsdn, &arena->large_mtx); } - if (config_prof && arena_prof_accum(tsdn, arena, usize)) { - prof_idump(tsdn); - } - - if (zero) { - assert(is_zeroed); - } else if (config_fill && unlikely(opt_junk_alloc)) { - memset(extent_addr_get(extent), JEMALLOC_ALLOC_JUNK, - extent_usize_get(extent)); - } arena_decay_tick(tsdn, arena); - return extent_addr_get(extent); + return edata_addr_get(edata); } -static void -large_dalloc_junk_impl(void *ptr, size_t size) { - memset(ptr, JEMALLOC_FREE_JUNK, size); -} -large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk = large_dalloc_junk_impl; - -static void -large_dalloc_maybe_junk_impl(void *ptr, size_t size) { - if (config_fill && have_dss && unlikely(opt_junk_free)) { - /* - * Only bother junk filling if the extent isn't about to be - * unmapped. - */ - if (opt_retain || (have_dss && extent_in_dss(ptr))) { - large_dalloc_junk(ptr, size); - } - } -} -large_dalloc_maybe_junk_t *JET_MUTABLE large_dalloc_maybe_junk = - large_dalloc_maybe_junk_impl; - static bool -large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { - arena_t *arena = extent_arena_get(extent); - size_t oldusize = extent_usize_get(extent); - extent_hooks_t *extent_hooks = extent_hooks_get(arena); - size_t diff = extent_size_get(extent) - (usize + sz_large_pad); +large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) { + arena_t *arena = arena_get_from_edata(edata); + ehooks_t *ehooks = arena_get_ehooks(arena); + size_t old_size = edata_size_get(edata); + size_t old_usize = edata_usize_get(edata); - assert(oldusize > usize); + assert(old_usize > usize); - if (extent_hooks->split == NULL) { + if (ehooks_split_will_fail(ehooks)) { return true; } - /* Split excess pages. */ - if (diff != 0) { - extent_t *trail = extent_split_wrapper(tsdn, arena, - &extent_hooks, extent, usize + sz_large_pad, - sz_size2index(usize), false, diff, SC_NSIZES, false); - if (trail == NULL) { - return true; - } - - if (config_fill && unlikely(opt_junk_free)) { - large_dalloc_maybe_junk(extent_addr_get(trail), - extent_size_get(trail)); - } - - arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, trail); + bool deferred_work_generated = false; + bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size, + usize + sz_large_pad, sz_size2index(usize), + &deferred_work_generated); + if (err) { + return true; } - - arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize); + if (deferred_work_generated) { + arena_handle_deferred_work(tsdn, arena); + } + arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize); return false; } static bool -large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, +large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize, bool zero) { - arena_t *arena = extent_arena_get(extent); - size_t oldusize = extent_usize_get(extent); - extent_hooks_t *extent_hooks = extent_hooks_get(arena); - size_t trailsize = usize - oldusize; + arena_t *arena = arena_get_from_edata(edata); - if (extent_hooks->merge == NULL) { - return true; - } + size_t old_size = edata_size_get(edata); + size_t old_usize = edata_usize_get(edata); + size_t new_size = usize + sz_large_pad; - if (config_fill && unlikely(opt_zero)) { - zero = true; - } - /* - * Copy zero into is_zeroed_trail and pass the copy when allocating the - * extent, so that it is possible to make correct junk/zero fill - * decisions below, even if is_zeroed_trail ends up true when zero is - * false. - */ - bool is_zeroed_trail = zero; - bool commit = true; - extent_t *trail; - bool new_mapping; - if ((trail = extents_alloc(tsdn, arena, &extent_hooks, - &arena->extents_dirty, extent_past_get(extent), trailsize, 0, - CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL - || (trail = extents_alloc(tsdn, arena, &extent_hooks, - &arena->extents_muzzy, extent_past_get(extent), trailsize, 0, - CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) { - if (config_stats) { - new_mapping = false; - } - } else { - if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks, - extent_past_get(extent), trailsize, 0, CACHELINE, false, - SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) { - return true; - } - if (config_stats) { - new_mapping = true; - } - } + szind_t szind = sz_size2index(usize); - if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) { - extent_dalloc_wrapper(tsdn, arena, &extent_hooks, trail); - return true; + bool deferred_work_generated = false; + bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size, + szind, zero, &deferred_work_generated); + + if (deferred_work_generated) { + arena_handle_deferred_work(tsdn, arena); } - rtree_ctx_t rtree_ctx_fallback; - rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - szind_t szind = sz_size2index(usize); - extent_szind_set(extent, szind); - rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, - (uintptr_t)extent_addr_get(extent), szind, false); - if (config_stats && new_mapping) { - arena_stats_mapped_add(tsdn, &arena->stats, trailsize); + if (err) { + return true; } if (zero) { - if (config_cache_oblivious) { + if (opt_cache_oblivious) { + assert(sz_large_pad == PAGE); /* * Zero the trailing bytes of the original allocation's * last page, since they are in an indeterminate state. @@ -197,28 +113,23 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, * of CACHELINE in [0 .. PAGE). */ void *zbase = (void *) - ((uintptr_t)extent_addr_get(extent) + oldusize); + ((uintptr_t)edata_addr_get(edata) + old_usize); void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase + PAGE)); size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase; assert(nzero > 0); memset(zbase, 0, nzero); } - assert(is_zeroed_trail); - } else if (config_fill && unlikely(opt_junk_alloc)) { - memset((void *)((uintptr_t)extent_addr_get(extent) + oldusize), - JEMALLOC_ALLOC_JUNK, usize - oldusize); } - - arena_extent_ralloc_large_expand(tsdn, arena, extent, oldusize); + arena_extent_ralloc_large_expand(tsdn, arena, edata, old_usize); return false; } bool -large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, +large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, size_t usize_max, bool zero) { - size_t oldusize = extent_usize_get(extent); + size_t oldusize = edata_usize_get(edata); /* The following should have been caught by callers. */ assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); @@ -228,16 +139,15 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, if (usize_max > oldusize) { /* Attempt to expand the allocation in-place. */ - if (!large_ralloc_no_move_expand(tsdn, extent, usize_max, + if (!large_ralloc_no_move_expand(tsdn, edata, usize_max, zero)) { - arena_decay_tick(tsdn, extent_arena_get(extent)); + arena_decay_tick(tsdn, arena_get_from_edata(edata)); return false; } /* Try again, this time with usize_min. */ if (usize_min < usize_max && usize_min > oldusize && - large_ralloc_no_move_expand(tsdn, extent, usize_min, - zero)) { - arena_decay_tick(tsdn, extent_arena_get(extent)); + large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) { + arena_decay_tick(tsdn, arena_get_from_edata(edata)); return false; } } @@ -247,14 +157,14 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, * the new size. */ if (oldusize >= usize_min && oldusize <= usize_max) { - arena_decay_tick(tsdn, extent_arena_get(extent)); + arena_decay_tick(tsdn, arena_get_from_edata(edata)); return false; } /* Attempt to shrink the allocation in-place. */ if (oldusize > usize_max) { - if (!large_ralloc_no_move_shrink(tsdn, extent, usize_max)) { - arena_decay_tick(tsdn, extent_arena_get(extent)); + if (!large_ralloc_no_move_shrink(tsdn, edata, usize_max)) { + arena_decay_tick(tsdn, arena_get_from_edata(edata)); return false; } } @@ -274,9 +184,9 @@ void * large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, size_t alignment, bool zero, tcache_t *tcache, hook_ralloc_args_t *hook_args) { - extent_t *extent = iealloc(tsdn, ptr); + edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); - size_t oldusize = extent_usize_get(extent); + size_t oldusize = edata_usize_get(edata); /* The following should have been caught by callers. */ assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ @@ -284,11 +194,11 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, && usize >= SC_LARGE_MINCLASS); /* Try to avoid moving the allocation. */ - if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) { + if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) { hook_invoke_expand(hook_args->is_realloc ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, usize, (uintptr_t)ptr, hook_args->args); - return extent_addr_get(extent); + return edata_addr_get(edata); } /* @@ -309,87 +219,104 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); size_t copysize = (usize < oldusize) ? usize : oldusize; - memcpy(ret, extent_addr_get(extent), copysize); - isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true); + memcpy(ret, edata_addr_get(edata), copysize); + isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true); return ret; } /* - * junked_locked indicates whether the extent's data have been junk-filled, and - * whether the arena's large_mtx is currently held. + * locked indicates whether the arena's large_mtx is currently held. */ static void -large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - bool junked_locked) { - if (!junked_locked) { +large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata, + bool locked) { + if (!locked) { /* See comments in arena_bin_slabs_full_insert(). */ if (!arena_is_auto(arena)) { malloc_mutex_lock(tsdn, &arena->large_mtx); - extent_list_remove(&arena->large, extent); + edata_list_active_remove(&arena->large, edata); malloc_mutex_unlock(tsdn, &arena->large_mtx); } - large_dalloc_maybe_junk(extent_addr_get(extent), - extent_usize_get(extent)); } else { /* Only hold the large_mtx if necessary. */ if (!arena_is_auto(arena)) { malloc_mutex_assert_owner(tsdn, &arena->large_mtx); - extent_list_remove(&arena->large, extent); + edata_list_active_remove(&arena->large, edata); } } - arena_extent_dalloc_large_prep(tsdn, arena, extent); + arena_extent_dalloc_large_prep(tsdn, arena, edata); } static void -large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { - extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; - arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, extent); +large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) { + bool deferred_work_generated = false; + pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated); + if (deferred_work_generated) { + arena_handle_deferred_work(tsdn, arena); + } } void -large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent) { - large_dalloc_prep_impl(tsdn, extent_arena_get(extent), extent, true); +large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata) { + large_dalloc_prep_impl(tsdn, arena_get_from_edata(edata), edata, true); } void -large_dalloc_finish(tsdn_t *tsdn, extent_t *extent) { - large_dalloc_finish_impl(tsdn, extent_arena_get(extent), extent); +large_dalloc_finish(tsdn_t *tsdn, edata_t *edata) { + large_dalloc_finish_impl(tsdn, arena_get_from_edata(edata), edata); } void -large_dalloc(tsdn_t *tsdn, extent_t *extent) { - arena_t *arena = extent_arena_get(extent); - large_dalloc_prep_impl(tsdn, arena, extent, false); - large_dalloc_finish_impl(tsdn, arena, extent); +large_dalloc(tsdn_t *tsdn, edata_t *edata) { + arena_t *arena = arena_get_from_edata(edata); + large_dalloc_prep_impl(tsdn, arena, edata, false); + large_dalloc_finish_impl(tsdn, arena, edata); arena_decay_tick(tsdn, arena); } size_t -large_salloc(tsdn_t *tsdn, const extent_t *extent) { - return extent_usize_get(extent); -} - -prof_tctx_t * -large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent) { - return extent_prof_tctx_get(extent); +large_salloc(tsdn_t *tsdn, const edata_t *edata) { + return edata_usize_get(edata); } void -large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx) { - extent_prof_tctx_set(extent, tctx); +large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, + bool reset_recent) { + assert(prof_info != NULL); + + prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata); + prof_info->alloc_tctx = alloc_tctx; + + if ((uintptr_t)alloc_tctx > (uintptr_t)1U) { + nstime_copy(&prof_info->alloc_time, + edata_prof_alloc_time_get(edata)); + prof_info->alloc_size = edata_prof_alloc_size_get(edata); + if (reset_recent) { + /* + * Reset the pointer on the recent allocation record, + * so that this allocation is recorded as released. + */ + prof_recent_alloc_reset(tsd, edata); + } + } } -void -large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) { - large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U); +static void +large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) { + edata_prof_tctx_set(edata, tctx); } -nstime_t -large_prof_alloc_time_get(const extent_t *extent) { - return extent_prof_alloc_time_get(extent); +void +large_prof_tctx_reset(edata_t *edata) { + large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U); } void -large_prof_alloc_time_set(extent_t *extent, nstime_t t) { - extent_prof_alloc_time_set(extent, t); +large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size) { + nstime_t t; + nstime_prof_init_update(&t); + edata_prof_alloc_time_set(edata, &t); + edata_prof_alloc_size_set(edata, size); + edata_prof_recent_alloc_init(edata); + large_prof_tctx_set(edata, tctx); } |