1 files changed, 113 insertions, 186 deletions

diff --git a/deps/jemalloc/src/large.c b/deps/jemalloc/src/large.c
index 8e7a781d3..5fc4bf584 100644
--- a/deps/jemalloc/src/large.c
+++ b/deps/jemalloc/src/large.c
@@ -1,11 +1,11 @@
-#define JEMALLOC_LARGE_C_
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/jemalloc_internal_includes.h"
 
 #include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/emap.h"
 #include "jemalloc/internal/extent_mmap.h"
 #include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/rtree.h"
+#include "jemalloc/internal/prof_recent.h"
 #include "jemalloc/internal/util.h"
 
 /******************************************************************************/
@@ -21,8 +21,7 @@ void *
 large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
     bool zero) {
 	size_t ausize;
-	extent_t *extent;
-	bool is_zeroed;
+	edata_t *edata;
 	UNUSED bool idump JEMALLOC_CC_SILENCE_INIT(false);
 
 	assert(!tsdn_null(tsdn) || arena != NULL);
@@ -32,163 +31,80 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
 		return NULL;
 	}
 
-	if (config_fill && unlikely(opt_zero)) {
-		zero = true;
-	}
-	/*
-	 * Copy zero into is_zeroed and pass the copy when allocating the
-	 * extent, so that it is possible to make correct junk/zero fill
-	 * decisions below, even if is_zeroed ends up true when zero is false.
-	 */
-	is_zeroed = zero;
 	if (likely(!tsdn_null(tsdn))) {
 		arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize);
 	}
-	if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn,
-	    arena, usize, alignment, &is_zeroed)) == NULL) {
+	if (unlikely(arena == NULL) || (edata = arena_extent_alloc_large(tsdn,
+	    arena, usize, alignment, zero)) == NULL) {
 		return NULL;
 	}
 
 	/* See comments in arena_bin_slabs_full_insert(). */
 	if (!arena_is_auto(arena)) {
-		/* Insert extent into large. */
+		/* Insert edata into large. */
 		malloc_mutex_lock(tsdn, &arena->large_mtx);
-		extent_list_append(&arena->large, extent);
+		edata_list_active_append(&arena->large, edata);
 		malloc_mutex_unlock(tsdn, &arena->large_mtx);
 	}
-	if (config_prof && arena_prof_accum(tsdn, arena, usize)) {
-		prof_idump(tsdn);
-	}
-
-	if (zero) {
-		assert(is_zeroed);
-	} else if (config_fill && unlikely(opt_junk_alloc)) {
-		memset(extent_addr_get(extent), JEMALLOC_ALLOC_JUNK,
-		    extent_usize_get(extent));
-	}
 
 	arena_decay_tick(tsdn, arena);
-	return extent_addr_get(extent);
+	return edata_addr_get(edata);
 }
 
-static void
-large_dalloc_junk_impl(void *ptr, size_t size) {
-	memset(ptr, JEMALLOC_FREE_JUNK, size);
-}
-large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk = large_dalloc_junk_impl;
-
-static void
-large_dalloc_maybe_junk_impl(void *ptr, size_t size) {
-	if (config_fill && have_dss && unlikely(opt_junk_free)) {
-		/*
-		 * Only bother junk filling if the extent isn't about to be
-		 * unmapped.
-		 */
-		if (opt_retain || (have_dss && extent_in_dss(ptr))) {
-			large_dalloc_junk(ptr, size);
-		}
-	}
-}
-large_dalloc_maybe_junk_t *JET_MUTABLE large_dalloc_maybe_junk =
-    large_dalloc_maybe_junk_impl;
-
 static bool
-large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
-	arena_t *arena = extent_arena_get(extent);
-	size_t oldusize = extent_usize_get(extent);
-	extent_hooks_t *extent_hooks = extent_hooks_get(arena);
-	size_t diff = extent_size_get(extent) - (usize + sz_large_pad);
+large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) {
+	arena_t *arena = arena_get_from_edata(edata);
+	ehooks_t *ehooks = arena_get_ehooks(arena);
+	size_t old_size = edata_size_get(edata);
+	size_t old_usize = edata_usize_get(edata);
 
-	assert(oldusize > usize);
+	assert(old_usize > usize);
 
-	if (extent_hooks->split == NULL) {
+	if (ehooks_split_will_fail(ehooks)) {
 		return true;
 	}
 
-	/* Split excess pages. */
-	if (diff != 0) {
-		extent_t *trail = extent_split_wrapper(tsdn, arena,
-		    &extent_hooks, extent, usize + sz_large_pad,
-		    sz_size2index(usize), false, diff, SC_NSIZES, false);
-		if (trail == NULL) {
-			return true;
-		}
-
-		if (config_fill && unlikely(opt_junk_free)) {
-			large_dalloc_maybe_junk(extent_addr_get(trail),
-			    extent_size_get(trail));
-		}
-
-		arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, trail);
+	bool deferred_work_generated = false;
+	bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size,
+	    usize + sz_large_pad, sz_size2index(usize),
+	    &deferred_work_generated);
+	if (err) {
+		return true;
 	}
-
-	arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize);
+	if (deferred_work_generated) {
+		arena_handle_deferred_work(tsdn, arena);
+	}
+	arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize);
 
 	return false;
 }
 
 static bool
-large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
+large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
     bool zero) {
-	arena_t *arena = extent_arena_get(extent);
-	size_t oldusize = extent_usize_get(extent);
-	extent_hooks_t *extent_hooks = extent_hooks_get(arena);
-	size_t trailsize = usize - oldusize;
+	arena_t *arena = arena_get_from_edata(edata);
 
-	if (extent_hooks->merge == NULL) {
-		return true;
-	}
+	size_t old_size = edata_size_get(edata);
+	size_t old_usize = edata_usize_get(edata);
+	size_t new_size = usize + sz_large_pad;
 
-	if (config_fill && unlikely(opt_zero)) {
-		zero = true;
-	}
-	/*
-	 * Copy zero into is_zeroed_trail and pass the copy when allocating the
-	 * extent, so that it is possible to make correct junk/zero fill
-	 * decisions below, even if is_zeroed_trail ends up true when zero is
-	 * false.
-	 */
-	bool is_zeroed_trail = zero;
-	bool commit = true;
-	extent_t *trail;
-	bool new_mapping;
-	if ((trail = extents_alloc(tsdn, arena, &extent_hooks,
-	    &arena->extents_dirty, extent_past_get(extent), trailsize, 0,
-	    CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL
-	    || (trail = extents_alloc(tsdn, arena, &extent_hooks,
-	    &arena->extents_muzzy, extent_past_get(extent), trailsize, 0,
-	    CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) {
-		if (config_stats) {
-			new_mapping = false;
-		}
-	} else {
-		if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks,
-		    extent_past_get(extent), trailsize, 0, CACHELINE, false,
-		    SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) {
-			return true;
-		}
-		if (config_stats) {
-			new_mapping = true;
-		}
-	}
+	szind_t szind = sz_size2index(usize);
 
-	if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {
-		extent_dalloc_wrapper(tsdn, arena, &extent_hooks, trail);
-		return true;
+	bool deferred_work_generated = false;
+	bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size,
+	    szind, zero, &deferred_work_generated);
+
+	if (deferred_work_generated) {
+		arena_handle_deferred_work(tsdn, arena);
 	}
-	rtree_ctx_t rtree_ctx_fallback;
-	rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
-	szind_t szind = sz_size2index(usize);
-	extent_szind_set(extent, szind);
-	rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,
-	    (uintptr_t)extent_addr_get(extent), szind, false);
 
-	if (config_stats && new_mapping) {
-		arena_stats_mapped_add(tsdn, &arena->stats, trailsize);
+	if (err) {
+		return true;
 	}
 
 	if (zero) {
-		if (config_cache_oblivious) {
+		if (opt_cache_oblivious) {
+			assert(sz_large_pad == PAGE);
 			/*
 			 * Zero the trailing bytes of the original allocation's
 			 * last page, since they are in an indeterminate state.
@@ -197,28 +113,23 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
 			 * of CACHELINE in [0 .. PAGE).
 			 */
 			void *zbase = (void *)
-			    ((uintptr_t)extent_addr_get(extent) + oldusize);
+			    ((uintptr_t)edata_addr_get(edata) + old_usize);
 			void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase +
 			    PAGE));
 			size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase;
 			assert(nzero > 0);
 			memset(zbase, 0, nzero);
 		}
-		assert(is_zeroed_trail);
-	} else if (config_fill && unlikely(opt_junk_alloc)) {
-		memset((void *)((uintptr_t)extent_addr_get(extent) + oldusize),
-		    JEMALLOC_ALLOC_JUNK, usize - oldusize);
 	}
-
-	arena_extent_ralloc_large_expand(tsdn, arena, extent, oldusize);
+	arena_extent_ralloc_large_expand(tsdn, arena, edata, old_usize);
 
 	return false;
 }
 
 bool
-large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
+large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
     size_t usize_max, bool zero) {
-	size_t oldusize = extent_usize_get(extent);
+	size_t oldusize = edata_usize_get(edata);
 
 	/* The following should have been caught by callers. */
 	assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS);
@@ -228,16 +139,15 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
 
 	if (usize_max > oldusize) {
 		/* Attempt to expand the allocation in-place. */
-		if (!large_ralloc_no_move_expand(tsdn, extent, usize_max,
+		if (!large_ralloc_no_move_expand(tsdn, edata, usize_max,
 		    zero)) {
-			arena_decay_tick(tsdn, extent_arena_get(extent));
+			arena_decay_tick(tsdn, arena_get_from_edata(edata));
 			return false;
 		}
 		/* Try again, this time with usize_min. */
 		if (usize_min < usize_max && usize_min > oldusize &&
-		    large_ralloc_no_move_expand(tsdn, extent, usize_min,
-		    zero)) {
-			arena_decay_tick(tsdn, extent_arena_get(extent));
+		    large_ralloc_no_move_expand(tsdn, edata, usize_min, zero)) {
+			arena_decay_tick(tsdn, arena_get_from_edata(edata));
 			return false;
 		}
 	}
@@ -247,14 +157,14 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
 	 * the new size.
 	 */
 	if (oldusize >= usize_min && oldusize <= usize_max) {
-		arena_decay_tick(tsdn, extent_arena_get(extent));
+		arena_decay_tick(tsdn, arena_get_from_edata(edata));
 		return false;
 	}
 
 	/* Attempt to shrink the allocation in-place. */
 	if (oldusize > usize_max) {
-		if (!large_ralloc_no_move_shrink(tsdn, extent, usize_max)) {
-			arena_decay_tick(tsdn, extent_arena_get(extent));
+		if (!large_ralloc_no_move_shrink(tsdn, edata, usize_max)) {
+			arena_decay_tick(tsdn, arena_get_from_edata(edata));
 			return false;
 		}
 	}
@@ -274,9 +184,9 @@ void *
 large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
     size_t alignment, bool zero, tcache_t *tcache,
     hook_ralloc_args_t *hook_args) {
-	extent_t *extent = iealloc(tsdn, ptr);
+	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
 
-	size_t oldusize = extent_usize_get(extent);
+	size_t oldusize = edata_usize_get(edata);
 	/* The following should have been caught by callers. */
 	assert(usize > 0 && usize <= SC_LARGE_MAXCLASS);
 	/* Both allocation sizes must be large to avoid a move. */
@@ -284,11 +194,11 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
 	    && usize >= SC_LARGE_MINCLASS);
 
 	/* Try to avoid moving the allocation. */
-	if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) {
+	if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) {
 		hook_invoke_expand(hook_args->is_realloc
 		    ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize,
 		    usize, (uintptr_t)ptr, hook_args->args);
-		return extent_addr_get(extent);
+		return edata_addr_get(edata);
 	}
 
 	/*
@@ -309,87 +219,104 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
 	    ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args);
 
 	size_t copysize = (usize < oldusize) ? usize : oldusize;
-	memcpy(ret, extent_addr_get(extent), copysize);
-	isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true);
+	memcpy(ret, edata_addr_get(edata), copysize);
+	isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true);
 	return ret;
 }
 
 /*
- * junked_locked indicates whether the extent's data have been junk-filled, and
- * whether the arena's large_mtx is currently held.
+ * locked indicates whether the arena's large_mtx is currently held.
  */
 static void
-large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
-    bool junked_locked) {
-	if (!junked_locked) {
+large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
+    bool locked) {
+	if (!locked) {
 		/* See comments in arena_bin_slabs_full_insert(). */
 		if (!arena_is_auto(arena)) {
 			malloc_mutex_lock(tsdn, &arena->large_mtx);
-			extent_list_remove(&arena->large, extent);
+			edata_list_active_remove(&arena->large, edata);
 			malloc_mutex_unlock(tsdn, &arena->large_mtx);
 		}
-		large_dalloc_maybe_junk(extent_addr_get(extent),
-		    extent_usize_get(extent));
 	} else {
 		/* Only hold the large_mtx if necessary. */
 		if (!arena_is_auto(arena)) {
 			malloc_mutex_assert_owner(tsdn, &arena->large_mtx);
-			extent_list_remove(&arena->large, extent);
+			edata_list_active_remove(&arena->large, edata);
 		}
 	}
-	arena_extent_dalloc_large_prep(tsdn, arena, extent);
+	arena_extent_dalloc_large_prep(tsdn, arena, edata);
 }
 
 static void
-large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
-	extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
-	arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, extent);
+large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
+	bool deferred_work_generated = false;
+	pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated);
+	if (deferred_work_generated) {
+		arena_handle_deferred_work(tsdn, arena);
+	}
 }
 
 void
-large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent) {
-	large_dalloc_prep_impl(tsdn, extent_arena_get(extent), extent, true);
+large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata) {
+	large_dalloc_prep_impl(tsdn, arena_get_from_edata(edata), edata, true);
 }
 
 void
-large_dalloc_finish(tsdn_t *tsdn, extent_t *extent) {
-	large_dalloc_finish_impl(tsdn, extent_arena_get(extent), extent);
+large_dalloc_finish(tsdn_t *tsdn, edata_t *edata) {
+	large_dalloc_finish_impl(tsdn, arena_get_from_edata(edata), edata);
 }
 
 void
-large_dalloc(tsdn_t *tsdn, extent_t *extent) {
-	arena_t *arena = extent_arena_get(extent);
-	large_dalloc_prep_impl(tsdn, arena, extent, false);
-	large_dalloc_finish_impl(tsdn, arena, extent);
+large_dalloc(tsdn_t *tsdn, edata_t *edata) {
+	arena_t *arena = arena_get_from_edata(edata);
+	large_dalloc_prep_impl(tsdn, arena, edata, false);
+	large_dalloc_finish_impl(tsdn, arena, edata);
 	arena_decay_tick(tsdn, arena);
 }
 
 size_t
-large_salloc(tsdn_t *tsdn, const extent_t *extent) {
-	return extent_usize_get(extent);
-}
-
-prof_tctx_t *
-large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent) {
-	return extent_prof_tctx_get(extent);
+large_salloc(tsdn_t *tsdn, const edata_t *edata) {
+	return edata_usize_get(edata);
 }
 
 void
-large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx) {
-	extent_prof_tctx_set(extent, tctx);
+large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info,
+    bool reset_recent) {
+	assert(prof_info != NULL);
+
+	prof_tctx_t *alloc_tctx = edata_prof_tctx_get(edata);
+	prof_info->alloc_tctx = alloc_tctx;
+
+	if ((uintptr_t)alloc_tctx > (uintptr_t)1U) {
+		nstime_copy(&prof_info->alloc_time,
+		    edata_prof_alloc_time_get(edata));
+		prof_info->alloc_size = edata_prof_alloc_size_get(edata);
+		if (reset_recent) {
+			/*
+			 * Reset the pointer on the recent allocation record,
+			 * so that this allocation is recorded as released.
+			 */
+			prof_recent_alloc_reset(tsd, edata);
+		}
+	}
 }
 
-void
-large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) {
-	large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
+static void
+large_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) {
+	edata_prof_tctx_set(edata, tctx);
 }
 
-nstime_t
-large_prof_alloc_time_get(const extent_t *extent) {
-	return extent_prof_alloc_time_get(extent);
+void
+large_prof_tctx_reset(edata_t *edata) {
+	large_prof_tctx_set(edata, (prof_tctx_t *)(uintptr_t)1U);
 }
 
 void
-large_prof_alloc_time_set(extent_t *extent, nstime_t t) {
-	extent_prof_alloc_time_set(extent, t);
+large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size) {
+	nstime_t t;
+	nstime_prof_init_update(&t);
+	edata_prof_alloc_time_set(edata, &t);
+	edata_prof_alloc_size_set(edata, size);
+	edata_prof_recent_alloc_init(edata);
+	large_prof_tctx_set(edata, tctx);
 }