1 files changed, 188 insertions, 112 deletions

diff --git a/deps/jemalloc/test/unit/rtree.c b/deps/jemalloc/test/unit/rtree.c
index b54b3e86f..908100fac 100644
--- a/deps/jemalloc/test/unit/rtree.c
+++ b/deps/jemalloc/test/unit/rtree.c
@@ -1,138 +1,207 @@
 #include "test/jemalloc_test.h"
 
+#include "jemalloc/internal/rtree.h"
+
+rtree_node_alloc_t *rtree_node_alloc_orig;
+rtree_node_dalloc_t *rtree_node_dalloc_orig;
+rtree_leaf_alloc_t *rtree_leaf_alloc_orig;
+rtree_leaf_dalloc_t *rtree_leaf_dalloc_orig;
+
+/* Potentially too large to safely place on the stack. */
+rtree_t test_rtree;
+
 static rtree_node_elm_t *
-node_alloc(size_t nelms)
-{
+rtree_node_alloc_intercept(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
+	rtree_node_elm_t *node;
+
+	if (rtree != &test_rtree) {
+		return rtree_node_alloc_orig(tsdn, rtree, nelms);
+	}
+
+	malloc_mutex_unlock(tsdn, &rtree->init_lock);
+	node = (rtree_node_elm_t *)calloc(nelms, sizeof(rtree_node_elm_t));
+	assert_ptr_not_null(node, "Unexpected calloc() failure");
+	malloc_mutex_lock(tsdn, &rtree->init_lock);
 
-	return ((rtree_node_elm_t *)calloc(nelms, sizeof(rtree_node_elm_t)));
+	return node;
 }
 
 static void
-node_dalloc(rtree_node_elm_t *node)
-{
+rtree_node_dalloc_intercept(tsdn_t *tsdn, rtree_t *rtree,
+    rtree_node_elm_t *node) {
+	if (rtree != &test_rtree) {
+		rtree_node_dalloc_orig(tsdn, rtree, node);
+		return;
+	}
 
 	free(node);
 }
 
-TEST_BEGIN(test_rtree_get_empty)
-{
-	unsigned i;
-
-	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
-		rtree_t rtree;
-		assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc),
-		    "Unexpected rtree_new() failure");
-		assert_ptr_null(rtree_get(&rtree, 0, false),
-		    "rtree_get() should return NULL for empty tree");
-		rtree_delete(&rtree);
+static rtree_leaf_elm_t *
+rtree_leaf_alloc_intercept(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
+	rtree_leaf_elm_t *leaf;
+
+	if (rtree != &test_rtree) {
+		return rtree_leaf_alloc_orig(tsdn, rtree, nelms);
 	}
+
+	malloc_mutex_unlock(tsdn, &rtree->init_lock);
+	leaf = (rtree_leaf_elm_t *)calloc(nelms, sizeof(rtree_leaf_elm_t));
+	assert_ptr_not_null(leaf, "Unexpected calloc() failure");
+	malloc_mutex_lock(tsdn, &rtree->init_lock);
+
+	return leaf;
 }
-TEST_END
 
-TEST_BEGIN(test_rtree_extrema)
-{
-	unsigned i;
-	extent_node_t node_a, node_b;
+static void
+rtree_leaf_dalloc_intercept(tsdn_t *tsdn, rtree_t *rtree,
+    rtree_leaf_elm_t *leaf) {
+	if (rtree != &test_rtree) {
+		rtree_leaf_dalloc_orig(tsdn, rtree, leaf);
+		return;
+	}
 
-	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
-		rtree_t rtree;
-		assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc),
-		    "Unexpected rtree_new() failure");
+	free(leaf);
+}
 
-		assert_false(rtree_set(&rtree, 0, &node_a),
-		    "Unexpected rtree_set() failure");
-		assert_ptr_eq(rtree_get(&rtree, 0, true), &node_a,
-		    "rtree_get() should return previously set value");
+TEST_BEGIN(test_rtree_read_empty) {
+	tsdn_t *tsdn;
 
-		assert_false(rtree_set(&rtree, ~((uintptr_t)0), &node_b),
-		    "Unexpected rtree_set() failure");
-		assert_ptr_eq(rtree_get(&rtree, ~((uintptr_t)0), true), &node_b,
-		    "rtree_get() should return previously set value");
+	tsdn = tsdn_fetch();
 
-		rtree_delete(&rtree);
-	}
+	rtree_t *rtree = &test_rtree;
+	rtree_ctx_t rtree_ctx;
+	rtree_ctx_data_init(&rtree_ctx);
+	assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure");
+	assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx, PAGE,
+	    false), "rtree_extent_read() should return NULL for empty tree");
+	rtree_delete(tsdn, rtree);
 }
 TEST_END
 
-TEST_BEGIN(test_rtree_bits)
-{
-	unsigned i, j, k;
-
-	for (i = 1; i < (sizeof(uintptr_t) << 3); i++) {
-		uintptr_t keys[] = {0, 1,
-		    (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)) - 1};
-		extent_node_t node;
-		rtree_t rtree;
-
-		assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc),
-		    "Unexpected rtree_new() failure");
-
-		for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) {
-			assert_false(rtree_set(&rtree, keys[j], &node),
-			    "Unexpected rtree_set() failure");
-			for (k = 0; k < sizeof(keys)/sizeof(uintptr_t); k++) {
-				assert_ptr_eq(rtree_get(&rtree, keys[k], true),
-				    &node, "rtree_get() should return "
-				    "previously set value and ignore "
-				    "insignificant key bits; i=%u, j=%u, k=%u, "
-				    "set key=%#"FMTxPTR", get key=%#"FMTxPTR, i,
-				    j, k, keys[j], keys[k]);
-			}
-			assert_ptr_null(rtree_get(&rtree,
-			    (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)), false),
-			    "Only leftmost rtree leaf should be set; "
-			    "i=%u, j=%u", i, j);
-			assert_false(rtree_set(&rtree, keys[j], NULL),
-			    "Unexpected rtree_set() failure");
-		}
+#undef NTHREADS
+#undef NITERS
+#undef SEED
 
-		rtree_delete(&rtree);
-	}
+TEST_BEGIN(test_rtree_extrema) {
+	extent_t extent_a, extent_b;
+	extent_init(&extent_a, NULL, NULL, LARGE_MINCLASS, false,
+	    sz_size2index(LARGE_MINCLASS), 0, extent_state_active, false,
+	    false, true);
+	extent_init(&extent_b, NULL, NULL, 0, false, NSIZES, 0,
+	    extent_state_active, false, false, true);
+
+	tsdn_t *tsdn = tsdn_fetch();
+
+	rtree_t *rtree = &test_rtree;
+	rtree_ctx_t rtree_ctx;
+	rtree_ctx_data_init(&rtree_ctx);
+	assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure");
+
+	assert_false(rtree_write(tsdn, rtree, &rtree_ctx, PAGE, &extent_a,
+	    extent_szind_get(&extent_a), extent_slab_get(&extent_a)),
+	    "Unexpected rtree_write() failure");
+	rtree_szind_slab_update(tsdn, rtree, &rtree_ctx, PAGE,
+	    extent_szind_get(&extent_a), extent_slab_get(&extent_a));
+	assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, PAGE, true),
+	    &extent_a,
+	    "rtree_extent_read() should return previously set value");
+
+	assert_false(rtree_write(tsdn, rtree, &rtree_ctx, ~((uintptr_t)0),
+	    &extent_b, extent_szind_get_maybe_invalid(&extent_b),
+	    extent_slab_get(&extent_b)), "Unexpected rtree_write() failure");
+	assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx,
+	    ~((uintptr_t)0), true), &extent_b,
+	    "rtree_extent_read() should return previously set value");
+
+	rtree_delete(tsdn, rtree);
 }
 TEST_END
 
-TEST_BEGIN(test_rtree_random)
-{
-	unsigned i;
-	sfmt_t *sfmt;
-#define	NSET 16
-#define	SEED 42
-
-	sfmt = init_gen_rand(SEED);
-	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
-		uintptr_t keys[NSET];
-		extent_node_t node;
-		unsigned j;
-		rtree_t rtree;
-
-		assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc),
-		    "Unexpected rtree_new() failure");
-
-		for (j = 0; j < NSET; j++) {
-			keys[j] = (uintptr_t)gen_rand64(sfmt);
-			assert_false(rtree_set(&rtree, keys[j], &node),
-			    "Unexpected rtree_set() failure");
-			assert_ptr_eq(rtree_get(&rtree, keys[j], true), &node,
-			    "rtree_get() should return previously set value");
-		}
-		for (j = 0; j < NSET; j++) {
-			assert_ptr_eq(rtree_get(&rtree, keys[j], true), &node,
-			    "rtree_get() should return previously set value");
+TEST_BEGIN(test_rtree_bits) {
+	tsdn_t *tsdn = tsdn_fetch();
+
+	uintptr_t keys[] = {PAGE, PAGE + 1,
+	    PAGE + (((uintptr_t)1) << LG_PAGE) - 1};
+
+	extent_t extent;
+	extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0,
+	    extent_state_active, false, false, true);
+
+	rtree_t *rtree = &test_rtree;
+	rtree_ctx_t rtree_ctx;
+	rtree_ctx_data_init(&rtree_ctx);
+	assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure");
+
+	for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) {
+		assert_false(rtree_write(tsdn, rtree, &rtree_ctx, keys[i],
+		    &extent, NSIZES, false),
+		    "Unexpected rtree_write() failure");
+		for (unsigned j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) {
+			assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx,
+			    keys[j], true), &extent,
+			    "rtree_extent_read() should return previously set "
+			    "value and ignore insignificant key bits; i=%u, "
+			    "j=%u, set key=%#"FMTxPTR", get key=%#"FMTxPTR, i,
+			    j, keys[i], keys[j]);
 		}
+		assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx,
+		    (((uintptr_t)2) << LG_PAGE), false),
+		    "Only leftmost rtree leaf should be set; i=%u", i);
+		rtree_clear(tsdn, rtree, &rtree_ctx, keys[i]);
+	}
 
-		for (j = 0; j < NSET; j++) {
-			assert_false(rtree_set(&rtree, keys[j], NULL),
-			    "Unexpected rtree_set() failure");
-			assert_ptr_null(rtree_get(&rtree, keys[j], true),
-			    "rtree_get() should return previously set value");
-		}
-		for (j = 0; j < NSET; j++) {
-			assert_ptr_null(rtree_get(&rtree, keys[j], true),
-			    "rtree_get() should return previously set value");
-		}
+	rtree_delete(tsdn, rtree);
+}
+TEST_END
+
+TEST_BEGIN(test_rtree_random) {
+#define NSET 16
+#define SEED 42
+	sfmt_t *sfmt = init_gen_rand(SEED);
+	tsdn_t *tsdn = tsdn_fetch();
+	uintptr_t keys[NSET];
+	rtree_t *rtree = &test_rtree;
+	rtree_ctx_t rtree_ctx;
+	rtree_ctx_data_init(&rtree_ctx);
+
+	extent_t extent;
+	extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0,
+	    extent_state_active, false, false, true);
+
+	assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure");
+
+	for (unsigned i = 0; i < NSET; i++) {
+		keys[i] = (uintptr_t)gen_rand64(sfmt);
+		rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree,
+		    &rtree_ctx, keys[i], false, true);
+		assert_ptr_not_null(elm,
+		    "Unexpected rtree_leaf_elm_lookup() failure");
+		rtree_leaf_elm_write(tsdn, rtree, elm, &extent, NSIZES, false);
+		assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx,
+		    keys[i], true), &extent,
+		    "rtree_extent_read() should return previously set value");
+	}
+	for (unsigned i = 0; i < NSET; i++) {
+		assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx,
+		    keys[i], true), &extent,
+		    "rtree_extent_read() should return previously set value, "
+		    "i=%u", i);
+	}
 
-		rtree_delete(&rtree);
+	for (unsigned i = 0; i < NSET; i++) {
+		rtree_clear(tsdn, rtree, &rtree_ctx, keys[i]);
+		assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx,
+		    keys[i], true),
+		   "rtree_extent_read() should return previously set value");
 	}
+	for (unsigned i = 0; i < NSET; i++) {
+		assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx,
+		    keys[i], true),
+		    "rtree_extent_read() should return previously set value");
+	}
+
+	rtree_delete(tsdn, rtree);
 	fini_gen_rand(sfmt);
 #undef NSET
 #undef SEED
@@ -140,12 +209,19 @@ TEST_BEGIN(test_rtree_random)
 TEST_END
 
 int
-main(void)
-{
-
-	return (test(
-	    test_rtree_get_empty,
+main(void) {
+	rtree_node_alloc_orig = rtree_node_alloc;
+	rtree_node_alloc = rtree_node_alloc_intercept;
+	rtree_node_dalloc_orig = rtree_node_dalloc;
+	rtree_node_dalloc = rtree_node_dalloc_intercept;
+	rtree_leaf_alloc_orig = rtree_leaf_alloc;
+	rtree_leaf_alloc = rtree_leaf_alloc_intercept;
+	rtree_leaf_dalloc_orig = rtree_leaf_dalloc;
+	rtree_leaf_dalloc = rtree_leaf_dalloc_intercept;
+
+	return test(
+	    test_rtree_read_empty,
 	    test_rtree_extrema,
 	    test_rtree_bits,
-	    test_rtree_random));
+	    test_rtree_random);
 }