erts: Add acful, abandon carried free utilization limit

This limit can be used to let erts_alloc know that it should
use MADV_FREE on any pages within a carrier that are currently
unused.

Before this change we have tried to have this as default, but that
caused performance issues as carriers oscillated in and out of the
migration pool. So in 25 we decided to remove it. However, this caused
MemAvailable from /proc/memory to become significantly lower and thus
machines that have a tight memory budget need this option.

So the solution is to add a new config option where the user can set
at which utilization limit that free pages in a carrier should be placed
in MemAvailable.

The option is by default set to 0, to mimic the behaviour of OTP 25.

1 files changed, 39 insertions, 19 deletions

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 7f3a9b420b..be5705c637 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1693,7 +1693,7 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr)
 }
 
 
-static UWord allctr_abandon_limit(Allctr_t *allctr);
+static UWord allctr_abandon_limit(Allctr_t *allctr, UWord);
 static void set_new_allctr_abandon_limit(Allctr_t*);
 static void abandon_carrier(Allctr_t*, Carrier_t*);
 static void poolify_my_carrier(Allctr_t*, Carrier_t*);
@@ -2251,7 +2251,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp)
     if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
 	return;
 
-    ASSERT(allctr->cpool.abandon_limit == allctr_abandon_limit(allctr));
+    ASSERT(allctr->cpool.abandon_limit == allctr_abandon_limit(allctr, allctr->cpool.util_limit));
     ASSERT(erts_thr_progress_is_managed_thread());
 
     if (allctr->cpool.disable_abandon)
@@ -2655,6 +2655,9 @@ carrier_mem_discard_free_blocks(Allctr_t *allocator, Carrier_t *carrier)
     Block_t *block;
     int i;
 
+    if (carrier->cpool.total_blocks_size > carrier->cpool.discard_limit)
+        return;
+
     block = allocator->first_fblk_in_mbc(allocator, carrier);
     i = 0;
 
@@ -2723,7 +2726,7 @@ mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp
 	blk = PREV_BLK(blk);
 	(*allctr->unlink_free_block)(allctr, blk);
 
-        if (discard) {
+        if (discard && crr->cpool.total_blocks_size <= crr->cpool.discard_limit) {
             mem_discard_coalesce(allctr, blk, &discard_region);
         }
 
@@ -2743,7 +2746,7 @@ mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp
 	    /* Coalesce with next block... */
 	    (*allctr->unlink_free_block)(allctr, nxt_blk);
 
-            if (discard) {
+            if (discard && crr->cpool.total_blocks_size <= crr->cpool.discard_limit) {
                 mem_discard_coalesce(allctr, nxt_blk, &discard_region);
             }
 
@@ -2783,7 +2786,7 @@ mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp
 	(*allctr->link_free_block)(allctr, blk);
 	HARD_CHECK_BLK_CARRIER(allctr, blk);
 
-        if (discard) {
+        if (discard && crr->cpool.total_blocks_size <= crr->cpool.discard_limit) {
             mem_discard_finish(allctr, blk, &discard_region);
         }
 
@@ -3866,6 +3869,17 @@ static void dealloc_my_carrier(Allctr_t *allctr, Carrier_t *crr)
     erts_alloc_ensure_handle_delayed_dealloc_call(allctr->ix);
 }
 
+static ERTS_INLINE UWord
+cpoll_init_carrier_limit(Carrier_t *crr, UWord percent_limit) {
+    UWord csz = CARRIER_SZ(crr);
+    UWord limit = percent_limit*csz;
+    if (limit > csz)
+        limit /= 100;
+    else
+        limit = (csz/100)*percent_limit;
+    return limit;
+}
+
 static ERTS_INLINE void
 cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 {
@@ -3878,16 +3892,12 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
     sys_memset(&crr->cpool.blocks_size, 0, sizeof(crr->cpool.blocks_size));
     sys_memset(&crr->cpool.blocks, 0, sizeof(crr->cpool.blocks));
     crr->cpool.total_blocks_size = 0;
-    if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr))
+    if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) {
 	crr->cpool.abandon_limit = 0;
-    else {
-	UWord csz = CARRIER_SZ(crr);
-	UWord limit = csz*allctr->cpool.util_limit;
-	if (limit > csz)
-	    limit /= 100;
-	else
-	    limit = (csz/100)*allctr->cpool.util_limit;
-	crr->cpool.abandon_limit = limit;
+        crr->cpool.discard_limit = 0;
+    } else {
+	crr->cpool.abandon_limit = cpoll_init_carrier_limit(crr, allctr->cpool.util_limit);
+	crr->cpool.discard_limit = cpoll_init_carrier_limit(crr, allctr->cpool.free_util_limit);
     }
     crr->cpool.state = ERTS_MBC_IS_HOME;
 }
@@ -3895,7 +3905,7 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr)
 
 
 static UWord
-allctr_abandon_limit(Allctr_t *allctr)
+allctr_abandon_limit(Allctr_t *allctr, UWord percent_limit)
 {
     UWord limit;
     UWord csz;
@@ -3906,11 +3916,11 @@ allctr_abandon_limit(Allctr_t *allctr)
         csz += allctr->mbcs.carriers[i].size;
     }
 
-    limit = csz*allctr->cpool.util_limit;
+    limit = csz*percent_limit;
     if (limit > csz)
 	limit /= 100;
     else
-	limit = (csz/100)*allctr->cpool.util_limit;
+	limit = (csz/100)*percent_limit;
 
     return limit;
 }
@@ -3918,7 +3928,7 @@ allctr_abandon_limit(Allctr_t *allctr)
 static void ERTS_INLINE
 set_new_allctr_abandon_limit(Allctr_t *allctr)
 {
-    allctr->cpool.abandon_limit = allctr_abandon_limit(allctr);
+    allctr->cpool.abandon_limit = allctr_abandon_limit(allctr, allctr->cpool.util_limit);
 }
 
 static void
@@ -4570,6 +4580,7 @@ static struct {
     Eterm smbcs;
     Eterm mbcgs;
     Eterm acul;
+    Eterm acful;
     Eterm acnl;
     Eterm acfml;
     Eterm cp;
@@ -4680,6 +4691,7 @@ init_atoms(Allctr_t *allctr)
 	AM_INIT(smbcs);
 	AM_INIT(mbcgs);
 	AM_INIT(acul);
+	AM_INIT(acful);
         AM_INIT(acnl);
         AM_INIT(acfml);
         AM_INIT(cp);
@@ -5439,7 +5451,7 @@ info_options(Allctr_t *allctr,
 	     Uint *szp)
 {
     Eterm res = THE_NON_VALUE;
-    UWord acul, acnl, acfml;
+    UWord acul, acful, acnl, acfml;
     char *cp_str;
     Eterm cp_atom;
 
@@ -5454,6 +5466,7 @@ info_options(Allctr_t *allctr,
     }
 
     acul = allctr->cpool.util_limit;
+    acful = allctr->cpool.free_util_limit;
     acnl = allctr->cpool.in_pool_limit;
     acfml = allctr->cpool.fblk_min_limit;
     ASSERT(allctr->cpool.carrier_pool <= ERTS_ALC_A_MAX);
@@ -5502,6 +5515,7 @@ info_options(Allctr_t *allctr,
 		   "option smbcs: %beu\n"
 		   "option mbcgs: %beu\n"
 		   "option acul: %bpu\n"
+		   "option acful: %bpu\n"
 		   "option acnl: %bpu\n"
 		   "option acfml: %bpu\n"
 		   "option cp: %s\n",
@@ -5524,6 +5538,7 @@ info_options(Allctr_t *allctr,
 		   allctr->smallest_mbc_size,
 		   allctr->mbc_growth_stages,
 		   acul,
+		   acful,
                    acnl,
                    acfml,
                    cp_str);
@@ -5543,6 +5558,9 @@ info_options(Allctr_t *allctr,
 	add_2tup(hpp, szp, &res,
 		 am.acul,
 		 bld_uint(hpp, szp, acul));
+        add_2tup(hpp, szp, &res,
+		 am.acful,
+		 bld_uint(hpp, szp, acful));
 	add_2tup(hpp, szp, &res,
 		 am.mbcgs,
 		 bld_uint(hpp, szp, allctr->mbc_growth_stages));
@@ -6795,6 +6813,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
         ASSERT(allctr->next_fblk_in_mbc);
 
         allctr->cpool.util_limit = init->acul;
+        allctr->cpool.free_util_limit = init->acful;
         allctr->cpool.in_pool_limit = init->acnl;
         allctr->cpool.fblk_min_limit = init->acfml;
         allctr->cpool.carrier_pool = init->cp;
@@ -6808,6 +6827,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
     }
     else {
         allctr->cpool.util_limit = 0;
+        allctr->cpool.free_util_limit = 0;
         allctr->cpool.in_pool_limit = 0;
         allctr->cpool.fblk_min_limit = 0;
         allctr->cpool.carrier_pool = -1;