Merge branch 'rickard/dirty-alloc-instances/OTP-17363' into maint

* rickard/dirty-alloc-instances/OTP-17363:
  Allocator instances for dirty schedulers

1 files changed, 222 insertions, 122 deletions

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index fcc80699ba..8313a4863e 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -137,6 +137,7 @@ Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
 Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0;
 Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0;
+int ERTS_WRITE_UNLIKELY(erts_no_aux_work_threads);
 
 static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0};
 int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS;
@@ -171,7 +172,12 @@ sched_get_busy_wait_params(ErtsSchedulerData *esdp)
     return &sched_busy_wait_params[esdp->type];
 }
 
-static ErtsAuxWorkData *aux_thread_aux_work_data;
+typedef union {
+    ErtsAuxWorkData data;
+    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAuxWorkData))];
+} ErtsAlignedAuxWorkData;
+
+static ErtsAlignedAuxWorkData *ERTS_WRITE_UNLIKELY(aligned_aux_work_data);
 
 #define ERTS_SCHDLR_SSPND_CHNG_NMSB		(((erts_aint32_t) 1) << 0)
 #define ERTS_SCHDLR_SSPND_CHNG_MSB		(((erts_aint32_t) 1) << 1)
@@ -485,8 +491,8 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist,
 				 ERTS_ALC_T_PROC_LIST)
 
 #define ERTS_SCHED_SLEEP_INFO_IX(IX)					\
-    (ASSERT(((int)-1) <= ((int) (IX))                 \
-            && ((int) (IX)) < ((int) erts_no_schedulers)),		\
+    (ASSERT((((int)-1) <= ((int) (IX)))					\
+            && (((int) (IX)) < (erts_no_aux_work_threads-1))),          \
      &aligned_sched_sleep_info[(IX)].ssi)
 #define ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(IX)				\
     (ASSERT(0 <= ((int) (IX))					        \
@@ -555,7 +561,7 @@ static void print_function_from_pc(fmtfn_t to, void *to_arg, ErtsCodePtr x);
 static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg);
 
 static void aux_work_timeout(void *unused);
-static void aux_work_timeout_early_init(int no_schedulers);
+static void aux_work_timeout_early_init(int max_no_aux_work_threads);
 static void setup_aux_work_timer(ErtsSchedulerData *esdp);
 
 static int execute_sys_tasks(Process *c_p,
@@ -1393,6 +1399,7 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable,
 
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
+                                          1,
 					  erts_no_schedulers,
 					  reply_sched_wall_time,
 					  (void *) swtrp);
@@ -1457,6 +1464,7 @@ Eterm erts_system_check_request(Process *c_p) {
 
     if (erts_no_schedulers > 1)
 	erts_schedule_multi_misc_aux_work(1,
+                                          1,
 					  erts_no_schedulers,
 					  reply_system_check,
 					  (void *) scrp);
@@ -1814,11 +1822,9 @@ init_misc_aux_work(void)
     misc_aux_work_queues = 
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q,
 					   sizeof(erts_algnd_misc_aux_work_q_t)
-					   * (erts_no_schedulers+1));
-
-    ix = 0; /* aux_thread + schedulers */
+					   * erts_no_aux_work_threads);
 
-    for (; ix <= erts_no_schedulers; ix++) {
+    for (ix = 0; ix < erts_no_aux_work_threads; ix++) {
 	qinit.arg = (void *) ERTS_SCHED_SLEEP_INFO_IX(ix-1);
 	erts_thr_q_initialize(&misc_aux_work_queues[ix].q, &qinit);
     }
@@ -1849,7 +1855,7 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp,
 		     erts_aint32_t aux_work,
 		     int waiting)
 {
-    ErtsThrQ_t *q = &misc_aux_work_queues[awdp->sched_id].q;
+    ErtsThrQ_t *q = &misc_aux_work_queues[awdp->aux_work_tid].q;
 
     unset_aux_work_flags_mb(awdp->ssi, ERTS_SSI_AUX_WORK_MISC);
     while (1) {
@@ -1877,23 +1883,36 @@ handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
 
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 
-    return misc_aux_work_clean(&misc_aux_work_queues[awdp->sched_id].q,
+    return misc_aux_work_clean(&misc_aux_work_queues[awdp->aux_work_tid].q,
 			       awdp,
 			       aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 }
 
+static ERTS_INLINE int
+get_aux_work_tid(void)
+{
+    /*
+     * All aux work threads are registered as delayed dealloc
+     * handlers, and their allocation index equals their tid.
+     */
+    ErtsThrAllocData *tadp = erts_get_thr_alloc_data();
+    if (!tadp || !tadp->delayed_dealloc_handler)
+        return -1; /* Not an aux work thread */
+    return tadp->alc_ix;
+}
 
 static ERTS_INLINE void
-schedule_misc_aux_work(int sched_id,
+schedule_misc_aux_work(int aux_work_tid,
 		       void (*func)(void *),
 		       void *arg)
 {
     ErtsThrQ_t *q;
     erts_misc_aux_work_t *mawp;
 
-    ASSERT(0 <= sched_id && sched_id <= erts_no_schedulers);
+    ASSERT(0 <= aux_work_tid
+	   && aux_work_tid < erts_no_aux_work_threads);
 
-    q = &misc_aux_work_queues[sched_id].q;
+    q = &misc_aux_work_queues[aux_work_tid].q;
     mawp = misc_aux_work_alloc();
     mawp->func = func;
     mawp->arg = arg;
@@ -1901,37 +1920,53 @@ schedule_misc_aux_work(int sched_id,
 }
 
 void
-erts_schedule_misc_aux_work(int sched_id,
+erts_schedule_misc_aux_work(int aux_work_tid,
 			    void (*func)(void *),
 			    void *arg)
 {
-    schedule_misc_aux_work(sched_id, func, arg);
+    schedule_misc_aux_work(aux_work_tid, func, arg);
 }
 
 void
 erts_schedule_multi_misc_aux_work(int ignore_self,
-				  int max_sched,
+                                  int min_tid,
+				  int max_tid,
 				  void (*func)(void *),
 				  void *arg)
 {
-    int id, self = 0;
-
-    if (ignore_self) {
-        ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    int tid, self;
 
-        /* ignore_self is meaningless on dirty schedulers since aux work can
-         * only run on normal schedulers, and their ids do not translate. */
-        if(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-            self = (int)esdp->no;
-        }
-    }
+    /*
+     * Threads handling misc aux work are:
+     * * Normal scheduler threads
+     * * Aux work threads
+     *
+     * Tids corresponds to threads as follows:
+     * * tid == 0
+     *     Standard aux thread. This thread is always
+     *     present on all systems.
+     * * 1 <= tid <= erts_no_schedulers
+     *      Normal scheduler threads. There are always at
+     *      least one normal scheduler. Tid equals scheduler
+     *      id.
+     * * erts_no_schedulers < tid < erts_no_aux_work_threads
+     *      Extra aux threads. Main purpose to handle
+     *      delayed dealloc for allocator instances for
+     *      dirty schedulers. May or may not exist. Maximum
+     *      amount of these threads equals amount of dirty
+     *      cpu schedulers.
+     */
 
-    ASSERT(0 < max_sched && max_sched <= erts_no_schedulers);
+    ASSERT(0 <= min_tid && min_tid < erts_no_aux_work_threads);
+    ASSERT(0 <= max_tid && max_tid < erts_no_aux_work_threads);
+    ASSERT(min_tid <= max_tid);
 
-    for (id = 1; id <= max_sched; id++) {
-	if (id == self)
+    self = ignore_self ? get_aux_work_tid() : -1;
+    
+    for (tid = min_tid; tid <= max_tid; tid++) {
+	if (tid == self)
 	    continue;
-	schedule_misc_aux_work(id, func, arg);
+	schedule_misc_aux_work(tid, func, arg);
    }
 }
 
@@ -2013,7 +2048,7 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
                                   | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC));
     aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
 		  | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC);
-    res = erts_alloc_fix_alloc_shrink(awdp->sched_id, aux_work);
+    res = erts_alloc_fix_alloc_shrink(awdp->aux_work_tid, aux_work);
     if (res) {
 	set_aux_work_flags(ssi, res);
 	aux_work |= res;
@@ -2060,10 +2095,10 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
 
     unset_aux_work_flags_mb(ssi, ERTS_SSI_AUX_WORK_DD);
     ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC);
-    erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
-						&need_thr_progress,
-						&wakeup,
-						&more_work);
+    erts_alloc_handle_delayed_dealloc(&awdp->alloc_data,
+				      &need_thr_progress,
+				      &wakeup,
+				      &more_work);
     ERTS_MSACC_POP_STATE_M_X();
     if (more_work) {
 	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD)
@@ -2103,10 +2138,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     need_thr_progress = 0;
     more_work = 0;
 
-    erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
-						&need_thr_progress,
-						&wakeup,
-						&more_work);
+    erts_alloc_handle_delayed_dealloc(&awdp->alloc_data,
+				      &need_thr_progress,
+				      &wakeup,
+				      &more_work);
     if (more_work) {
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
@@ -2347,13 +2382,24 @@ setup_thr_debug_wait_completed(void *vproc)
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     ErtsAuxWorkData *awdp;
     erts_aint32_t wait_flags, aux_work_flags;
-    awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data;
+
+    if (esdp)
+        awdp = &esdp->aux_work_data; /* A normal scheduler... */
+    else {
+        /* An aux thread... */
+        ErtsThrAllocData *tadp = erts_get_thr_alloc_data();
+        char *ptr;
+        ASSERT(tadp);
+        ptr = (char *) tadp;
+        ptr -= offsetof(ErtsAuxWorkData, alloc_data);
+        awdp = (ErtsAuxWorkData *) ptr;
+    }
 
     wait_flags = 0;
     aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
 
     if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) {
-	erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
+	erts_alloc_fix_alloc_shrink(awdp->aux_work_tid, 0);
 	wait_flags |= (ERTS_SSI_AUX_WORK_DD
 		       | ERTS_SSI_AUX_WORK_DD_THR_PRGR);
 	aux_work_flags |= ERTS_SSI_AUX_WORK_DD;
@@ -2383,19 +2429,15 @@ static void later_thr_debug_wait_completed(void *vlop)
     struct debug_lop *lop = vlop;
 
     if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 1) {
-        erts_aint32_t count = (erts_aint32_t) erts_no_schedulers;
-        count += 1; /* aux thread */
+        erts_aint32_t count = (erts_aint32_t) erts_no_aux_work_threads;
         erts_atomic32_set_nob(&debug_wait_completed_count, count);
 
-        /* scheduler threads */
+        /* All scheduler threads as well as all aux threads... */
         erts_schedule_multi_misc_aux_work(0,
-                                          erts_no_schedulers,
+                                          0,
+                                          erts_no_aux_work_threads-1,
                                           setup_thr_debug_wait_completed,
                                           lop->proc);
-        /* aux_thread */
-        erts_schedule_misc_aux_work(0,
-                                    setup_thr_debug_wait_completed,
-                                    lop->proc);
     }
     erts_free(ERTS_ALC_T_DEBUG, lop);
 }
@@ -2435,6 +2477,7 @@ erts_debug_wait_completed(Process *c_p, int flags)
 
         /* First flush later-ops on all scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
+                                          1,
 					  erts_no_schedulers,
 					  init_thr_debug_wait_completed,
 					  (void *) c_p);
@@ -2507,11 +2550,27 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 }
 
 void
-erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp)
+erts_more_yield_aux_work(ErtsAuxWorkData *awdp)
 {
-    ASSERT(esdp == erts_get_scheduler_data());
-    /* Always called by the scheduler itself... */
-    set_aux_work_flags_wakeup_nob(esdp->ssi, ERTS_SSI_AUX_WORK_YIELD);
+    /* Should *always* be called by the aux-work thread itself... */
+    ASSERT(awdp && awdp->aux_work_tid == get_aux_work_tid());
+    set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_YIELD);
+}
+
+ErtsAuxWorkData *
+erts_get_aux_work_data(void)
+{
+    ErtsAuxWorkData *awdp;
+    int tid = get_aux_work_tid();
+    ASSERT(0 <= tid && tid < erts_no_aux_work_threads);
+    if (tid < 1)
+        awdp = tid == 0 ? &aligned_aux_work_data[0].data : NULL;
+    else if (tid <= erts_no_schedulers)
+        awdp = &erts_aligned_scheduler_data[tid-1].esd.aux_work_data;
+    else
+        awdp = &aligned_aux_work_data[tid - (int) erts_no_schedulers].data;
+    ASSERT(!awdp || awdp->aux_work_tid == tid);
+    return awdp;
 }
 
 static ERTS_INLINE erts_aint32_t
@@ -2531,10 +2590,8 @@ handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 
     /* Various yielding operations... */
 
-    yield |= erts_handle_yielded_ets_all_request(awdp->esdp,
-                                                 &awdp->yield.ets_all);
-    yield |= erts_handle_yielded_alcu_blockscan(awdp->esdp,
-                                                &awdp->yield.alcu_blockscan);
+    yield |= erts_handle_yielded_ets_all_request(awdp);
+    yield |= erts_handle_yielded_alcu_blockscan(awdp);
 
     /*
      * Other yielding operations...
@@ -2718,7 +2775,7 @@ start_aux_work_timer(ErtsSchedulerData *esdp)
 }
 
 static void
-aux_work_timeout_early_init(int no_schedulers)
+aux_work_timeout_early_init(int max_no_aux_work_threads)
 {
     int i;
     UWord p;
@@ -2729,7 +2786,8 @@ aux_work_timeout_early_init(int no_schedulers)
      */
 
     p = (UWord) malloc((sizeof(ErtsAuxWorkTmo)
-			+ sizeof(erts_atomic32_t)*(no_schedulers+1))
+			+ (sizeof(erts_atomic32_t)
+                           * max_no_aux_work_threads))
 		       + ERTS_CACHE_LINE_SIZE-1);
     if (!p) {
         ERTS_INTERNAL_ERROR("malloc failed to allocate memory!");
@@ -2744,7 +2802,7 @@ aux_work_timeout_early_init(int no_schedulers)
 #ifdef DEBUG
     erts_atomic32_init_nob(&aux_work_tmo->used, 0);
 #endif
-    for (i = 0; i <= no_schedulers; i++)
+    for (i = 0; i < max_no_aux_work_threads; i++)
 	erts_atomic32_init_nob(&aux_work_tmo->type[i], 0);
 }
 
@@ -3079,37 +3137,49 @@ thr_prgr_fin_wait(void *vssi)
 				      | ERTS_SSI_FLG_TSE_SLEEPING));
 }
 
-static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp);
+static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp,
+                               char *dawwp);
 
 void
-erts_aux_thread_poke()
+erts_aux_thread_poke(void)
 {
     erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1));
 }
 
 static void *
-aux_thread(void *unused)
+aux_thread(void *vix)
 {
-    ErtsAuxWorkData *awdp = aux_thread_aux_work_data;
-    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(-1);
+    int ix = (int) (Sint) vix;
+    int id = ix == 0 ? 1 : ix + 1 - erts_no_schedulers;
+    ErtsAuxWorkData *awdp = &aligned_aux_work_data[id-1].data;
+    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix-1);
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
     ErtsThrPrgrData *tpd;
     int thr_prgr_active = 1;
     ERTS_MSACC_DECLARE_CACHE();
 
+    ASSERT(ix == 0
+           || (erts_no_schedulers < ix
+               && ix < erts_no_aux_work_threads));
+    ASSERT(0 < id && id <= 1 + erts_no_dirty_alloc_instances);
+    
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
-	char buf[] = "aux_thread";
-	erts_lc_set_thread_name(buf);
+        char buf[31];
+        erts_snprintf(&buf[0], 31, "aux-thread %d", id);
+        erts_lc_set_thread_name(&buf[0]);
     }
 #endif
 
-    erts_port_task_pre_alloc_init_thread();
+    if (ix == 0)
+        erts_port_task_pre_alloc_init_thread();
     ssi->event = erts_tse_fetch();
     erts_tse_return(ssi->event);
 
-    erts_msacc_init_thread("aux", 1, 1);
+    erts_msacc_init_thread("aux", id, 1);
+
+    erts_alloc_register_delayed_dealloc_handler_thread(&awdp->alloc_data, ix);
 
     callbacks.arg = (void *) ssi;
     callbacks.wakeup = thr_prgr_wakeup;
@@ -3120,13 +3190,16 @@ aux_thread(void *unused)
     tpd = erts_thr_progress_register_managed_thread(NULL, &callbacks, 1, 0);
     init_aux_work_data(awdp, NULL, NULL);
     awdp->ssi = ssi;
+    awdp->aux_work_tid = ix;
 
 #if ERTS_POLL_USE_FALLBACK
+    if (ix == 0) {
 #if ERTS_POLL_USE_SCHEDULER_POLLING
-    ssi->psi = erts_create_pollset_thread(-2, tpd);
+	ssi->psi = erts_create_pollset_thread(-2, tpd);
 #else
-    ssi->psi = erts_create_pollset_thread(-1, tpd);
+	ssi->psi = erts_create_pollset_thread(-1, tpd);
 #endif
+    }
 #endif
 
     sched_prep_spin_wait(ssi);
@@ -3149,7 +3222,7 @@ aux_thread(void *unused)
 	if (!aux_work) {
 
 #ifdef ERTS_BREAK_REQUESTED
-            if (ERTS_BREAK_REQUESTED)
+            if (ix == 0 && ERTS_BREAK_REQUESTED)
                 erts_do_break_handling();
 #endif
 
@@ -3157,40 +3230,43 @@ aux_thread(void *unused)
 		erts_thr_progress_active(tpd, thr_prgr_active = 0);
 
 #if ERTS_POLL_USE_FALLBACK
+            if (ix == 0) {
 
-	    flgs = sched_spin_wait(ssi, 0);
+                flgs = sched_spin_wait(ssi, 0);
 
-	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
-		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		    ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
-		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-                    erts_check_io(ssi->psi, ERTS_POLL_INF_TIMEOUT, 0);
-		}
+                if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING);
+                    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                        ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING);
+                        ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                        erts_check_io(ssi->psi, ERTS_POLL_INF_TIMEOUT, 0);
+                    }
+                }
             }
-#else
-            erts_thr_progress_prepare_wait(tpd);
-
-	    flgs = sched_spin_wait(ssi, 0);
+            else 
+#endif
+            {
+                erts_thr_progress_prepare_wait(tpd);
+		flgs = sched_spin_wait(ssi, 0);
 
-	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
-		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-                    int res;
-		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
-		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-                    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
-                    do {
-                        res = erts_tse_wait(ssi->event);
-                    } while (res == EINTR);
-                    ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
-		}
-                erts_tse_return(ssi->event);
+                if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+                    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+                        int res;
+                        ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+                        ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+                        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
+                        do {
+                            res = erts_tse_wait(ssi->event);
+                        } while (res == EINTR);
+                        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
+                    }
+                    erts_tse_return(ssi->event);
+                }
+                erts_thr_progress_finalize_wait(tpd);
             }
-            erts_thr_progress_finalize_wait(tpd);
-#endif
-	}
+        }
 
 	flgs = sched_prep_spin_wait(ssi);
     }
@@ -5609,11 +5685,11 @@ runq_supervisor(void *unused)
 
 
 void
-erts_early_init_scheduling(int no_schedulers)
+erts_early_init_scheduling(int max_no_aux_work_threads)
 {
     ErtsSchedType type;
 
-    aux_work_timeout_early_init(no_schedulers);
+    aux_work_timeout_early_init(max_no_aux_work_threads);
 
     for (type = ERTS_SCHED_TYPE_FIRST; type <= ERTS_SCHED_TYPE_LAST; type++) {
         erts_sched_set_wakeup_other_threshold(type, "medium");
@@ -5723,7 +5799,8 @@ erts_sched_set_wake_cleanup_threshold(char *str)
 }
 
 static void
-init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
+init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp,
+                   char *dawwp)
 {
     int id = 0;
     if (esdp) {
@@ -5746,7 +5823,7 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
         }
     }
 
-    awdp->sched_id = id;
+    awdp->aux_work_tid = id;
     awdp->esdp = esdp;
     awdp->ssi = esdp ? esdp->ssi : NULL;
     awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST;
@@ -5769,12 +5846,13 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     else {
 	int i;
 	awdp->delayed_wakeup.job = (ErtsDelayedAuxWorkWakeupJob *) dawwp;
-	dawwp += sizeof(ErtsDelayedAuxWorkWakeupJob)*(erts_no_schedulers+1);
+	dawwp += sizeof(ErtsDelayedAuxWorkWakeupJob)*erts_no_aux_work_threads;
 	awdp->delayed_wakeup.sched2jix = (int *) dawwp;
 	awdp->delayed_wakeup.jix = -1;
-	for (i = 0; i <= erts_no_schedulers; i++)
+	for (i = 0; i < erts_no_aux_work_threads; i++)
 	    awdp->delayed_wakeup.sched2jix[i] = -1;
     }
+    erts_alcu_blockscan_init(awdp);
     awdp->debug.wait_completed.flags = 0;
     awdp->debug.wait_completed.callback = NULL;
     awdp->debug.wait_completed.arg = NULL;
@@ -5896,7 +5974,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 		     int no_dirty_io_schedulers
 		     )
 {
-    int ix, n, no_ssi, tot_rqs;
+    int ix, n, tot_rqs;
     char *daww_ptr;
     size_t daww_sz;
     size_t size_runqs;
@@ -6019,13 +6097,19 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
     erts_no_dirty_io_schedulers = no_dirty_io_schedulers;
     erts_no_total_schedulers += no_dirty_io_schedulers;
 
+    /* normal schedulers */
+    erts_no_aux_work_threads = n;
+    /* standard aux thread */
+    erts_no_aux_work_threads += 1;
+    /* extra aux threads */
+    erts_no_aux_work_threads += erts_no_dirty_alloc_instances;
+
     /* Create and initialize scheduler sleep info */
-    no_ssi = n + 1 /* aux thread */;
     aligned_sched_sleep_info =
 	erts_alloc_permanent_cache_aligned(
 	    ERTS_ALC_T_SCHDLR_SLP_INFO,
-	    no_ssi*sizeof(ErtsAlignedSchedulerSleepInfo));
-    for (ix = 0; ix < no_ssi; ix++) {
+	    erts_no_aux_work_threads*sizeof(ErtsAlignedSchedulerSleepInfo));
+    for (ix = 0; ix < erts_no_aux_work_threads; ix++) {
 	ErtsSchedulerSleepInfo *ssi = &aligned_sched_sleep_info[ix].ssi;
 #if 0 /* no need to initialize these... */
 	ssi->next = NULL;
@@ -6067,7 +6151,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
     /* Create and initialize scheduler specific data */
 
     daww_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE((sizeof(ErtsDelayedAuxWorkWakeupJob)
-					      + sizeof(int))*(n+1));
+					      + sizeof(int))*erts_no_aux_work_threads);
     daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 						  daww_sz*n);
 
@@ -6079,7 +6163,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
 	init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix),
 			    ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz,
-			    NULL, 0);
+                            NULL, 0);
     }
 
     {
@@ -6109,7 +6193,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
 	    init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix),
 				ERTS_DIRTY_IO_RUNQ, NULL, 0,
-				&adsp[adspix++].dsp, ts);
+                                &adsp[adspix++].dsp, ts);
 	}
     }
 
@@ -6120,9 +6204,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online, int no_poll_th
     erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
     debug_wait_completed_flags = 0;
 
-    aux_thread_aux_work_data =
+    aligned_aux_work_data =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
-					   sizeof(ErtsAuxWorkData));
+					   sizeof(ErtsAlignedAuxWorkData)
+					   * (erts_no_dirty_alloc_instances + 1));
 
     init_no_runqs(no_schedulers_online, no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
@@ -8539,7 +8624,6 @@ sched_thread_func(void *vesdp)
     esdp->ssi->psi = erts_create_pollset_thread(-1, NULL);
 #endif
 
-    erts_alloc_register_scheduler(vesdp);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
 	char buf[31];
@@ -8548,6 +8632,7 @@ sched_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
+    erts_alloc_register_scheduler(vesdp);
 #if HAVE_ERTS_MSEG
     erts_mseg_late_init();
 #endif
@@ -8564,7 +8649,6 @@ sched_thread_func(void *vesdp)
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
 #endif
 
-    erts_alcu_sched_spec_data_init(esdp);
     erts_ets_sched_spec_data_init(esdp);
     erts_utils_sched_spec_data_init();
 
@@ -8612,6 +8696,7 @@ sched_dirty_cpu_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
+    erts_alloc_register_scheduler(vesdp);
     esdp->aux_work_data.async_ready.queue = NULL;
 
     erts_proc_lock_prepare_proc_lock_waiter();
@@ -8659,6 +8744,7 @@ sched_dirty_io_thread_func(void *vesdp)
     }
 #endif
     erts_tsd_set(sched_data_key, vesdp);
+    erts_alloc_register_scheduler(vesdp);
     esdp->aux_work_data.async_ready.queue = NULL;
 
     erts_proc_lock_prepare_proc_lock_waiter();
@@ -8737,11 +8823,19 @@ erts_start_schedulers(void)
 	}
     }
 
-    erts_snprintf(opts.name, sizeof(name), "aux");
+    ix = 0;
+    while (ix < erts_no_aux_work_threads) {
+	int id = ix == 0 ? 1 : ix + 1 - (int) erts_no_schedulers;
+	erts_snprintf(opts.name, sizeof(name), "%d_aux", id);
 
-    res = ethr_thr_create(&tid, aux_thread, NULL, &opts);
-    if (res != 0)
-	erts_exit(ERTS_ABORT_EXIT, "Failed to create aux thread, error = %d\n", res);
+	res = ethr_thr_create(&tid, aux_thread, (void *) (Sint) ix, &opts);
+	if (res != 0)
+	    erts_exit(ERTS_ABORT_EXIT, "Failed to create aux thread %d, error = %d\n", res);
+	if (ix == 0)
+	    ix = (int) (1 + erts_no_schedulers);
+	else
+	    ix++;
+    }
 
     block_poll_thread_data = (ErtsAlignedBlockPollThreadData *)
         erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BLOCK_PTHR_DATA,
@@ -8977,6 +9071,12 @@ erts_internal_is_process_executing_dirty_1(BIF_ALIST_1)
     BIF_RET(am_false);
 }
 
+BIF_RETTYPE
+erts_internal_no_aux_work_threads_0(BIF_ALIST_0)
+{
+    BIF_RET(make_small(erts_no_aux_work_threads));
+}
+
 static ERTS_INLINE void
 run_queues_len_aux(ErtsRunQueue *rq, Uint *tot_len, Uint *qlen, int *ip, int incl_active_sched, int locked)
 {