1 files changed, 207 insertions, 110 deletions

diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c
index 8f393f92cf..3888c72363 100644
--- a/erts/emulator/beam/code_ix.c
+++ b/erts/emulator/beam/code_ix.c
@@ -46,20 +46,24 @@ erts_atomic32_t outstanding_blocking_code_barriers;
 erts_atomic32_t the_active_code_index;
 erts_atomic32_t the_staging_code_index;
 
-static int code_writing_seized = 0;
-static Process* code_writing_process = NULL;
-struct code_write_queue_item {
-    Process *p;
-    void (*aux_func)(void *);
-    void *aux_arg;
-    struct code_write_queue_item* next;
+struct code_permission {
+    erts_mtx_t lock;
+
+    ErtsSchedulerData *scheduler;
+    Process *owner;
+
+    int seized;
+    struct code_permission_queue_item {
+        Process *p;
+        void (*aux_func)(void *);
+        void *aux_arg;
+
+        struct code_permission_queue_item *next;
+    } *queue;
 };
-static struct code_write_queue_item* code_write_queue = NULL;
-static erts_mtx_t code_write_permission_mtx;
 
-#ifdef ERTS_ENABLE_LOCK_CHECK
-static erts_tsd_key_t has_code_write_permission;
-#endif
+static struct code_permission code_mod_permission = {0};
+static struct code_permission code_stage_permission = {0};
 
 #ifdef DEBUG
 static erts_tsd_key_t needs_code_barrier;
@@ -74,12 +78,14 @@ void erts_code_ix_init(void)
     erts_atomic32_init_nob(&outstanding_blocking_code_barriers, 0);
     erts_atomic32_init_nob(&the_active_code_index, 0);
     erts_atomic32_init_nob(&the_staging_code_index, 0);
-    erts_mtx_init(&code_write_permission_mtx, "code_write_permission", NIL,
+
+    erts_mtx_init(&code_mod_permission.lock,
+        "code_mod_permission", NIL,
         ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_tsd_key_create(&has_code_write_permission,
-                        "erts_has_code_write_permission");
-#endif
+    erts_mtx_init(&code_stage_permission.lock,
+        "code_stage_permission", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
+
 #ifdef DEBUG
     erts_tsd_key_create(&needs_code_barrier,
                         "erts_needs_code_barrier");
@@ -90,16 +96,17 @@ void erts_code_ix_init(void)
 void erts_start_staging_code_ix(int num_new)
 {
     beam_catches_start_staging();
+    erts_fun_start_staging();
     export_start_staging();
     module_start_staging();
     erts_start_staging_ranges(num_new);
     CIX_TRACE("start");
 }
 
-
 void erts_end_staging_code_ix(void)
 {
     beam_catches_end_staging(1);
+    erts_fun_end_staging(1);
     export_end_staging(1);
     module_end_staging(1);
     erts_end_staging_ranges(1);
@@ -123,122 +130,235 @@ void erts_commit_staging_code_ix(void)
 void erts_abort_staging_code_ix(void)
 {
     beam_catches_end_staging(0);
+    erts_fun_end_staging(0);
     export_end_staging(0);
     module_end_staging(0);
     erts_end_staging_ranges(0);
     CIX_TRACE("abort");
 }
 
-static int try_seize_cwp(Process* c_p,
-                         void (*aux_func)(void *),
-                         void *aux_arg);
-
 #if defined(DEBUG) || defined(ADDRESS_SANITIZER)
 #    define CWP_DBG_FORCE_TRAP
 #endif
 
-/*
- * Calller _must_ yield if we return 0
- */
-int erts_try_seize_code_write_permission(Process* c_p)
-{
-    ASSERT(c_p != NULL);
-
-#ifdef CWP_DBG_FORCE_TRAP
-    if (!(c_p->flags & F_DBG_FORCED_TRAP)) {
-        c_p->flags |= F_DBG_FORCED_TRAP;
-        return 0;
-    } else {
-        /* back from forced trap */
-        c_p->flags &= ~F_DBG_FORCED_TRAP;
-    }
+#ifdef ERTS_ENABLE_LOCK_CHECK
+static int has_code_permission(struct code_permission *lock);
 #endif
 
-    return try_seize_cwp(c_p, NULL, NULL);
-}
-
-int erts_try_seize_code_write_permission_aux(void (*aux_func)(void *),
-                                             void *aux_arg)
-{
-    ASSERT(aux_func != NULL);
-    return try_seize_cwp(NULL, aux_func, aux_arg);
-}
-
-static int try_seize_cwp(Process* c_p,
-                         void (*aux_func)(void *),
-                         void *aux_arg)
+static int try_seize_code_permission(struct code_permission *perm,
+                                     Process* c_p,
+                                     void (*aux_func)(void *),
+                                     void *aux_arg)
 {
     int success;
-    ASSERT(!erts_thr_progress_is_blocking()); /* to avoid deadlock */
 
-    erts_mtx_lock(&code_write_permission_mtx);
-    success = !code_writing_seized;
+    ASSERT(!erts_thr_progress_is_blocking()); /* To avoid deadlock */
+
+    erts_mtx_lock(&perm->lock);
+    success = !perm->seized;
+
     if (success) {
-        code_writing_seized = 1;
-	code_writing_process = c_p;
-#ifdef ERTS_ENABLE_LOCK_CHECK
-	erts_tsd_set(has_code_write_permission, (void *) 1);
-#endif
-    }
-    else { /* Already locked */
-	struct code_write_queue_item* qitem;
+        if (c_p == NULL) {
+            ASSERT(aux_func);
+            perm->scheduler = erts_get_scheduler_data();
+        }
+
+        perm->owner = c_p;
+        perm->seized = 1;
+    } else { /* Already locked */
+        struct code_permission_queue_item* qitem;
+
         qitem = erts_alloc(ERTS_ALC_T_CODE_IX_LOCK_Q, sizeof(*qitem));
         if (c_p) {
-            ASSERT(code_writing_process != c_p);
+            ERTS_LC_ASSERT(perm->owner != c_p);
+
             qitem->p = c_p;
             qitem->aux_func = NULL;
             qitem->aux_arg = NULL;
             erts_proc_inc_refc(c_p);
             erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
-        }
-        else {
+        } else {
             qitem->p = NULL;
             qitem->aux_func = aux_func;
             qitem->aux_arg = aux_arg;
         }
-        qitem->next = code_write_queue;
-        code_write_queue = qitem;
+
+        qitem->next = perm->queue;
+        perm->queue = qitem;
     }
-   erts_mtx_unlock(&code_write_permission_mtx);
-   return success;
+
+    erts_mtx_unlock(&perm->lock);
+
+    return success;
 }
 
-void erts_release_code_write_permission(void)
-{
-    erts_mtx_lock(&code_write_permission_mtx);
-    ERTS_LC_ASSERT(erts_has_code_write_permission());
-    while (code_write_queue != NULL) { /* unleash the entire herd */
-	struct code_write_queue_item* qitem = code_write_queue;
+static void release_code_permission(struct code_permission *perm) {
+    ERTS_LC_ASSERT(has_code_permission(perm));
+
+    erts_mtx_lock(&perm->lock);
+
+    /* Unleash the entire herd */
+    while (perm->queue != NULL) {
+        struct code_permission_queue_item* qitem = perm->queue;
+
         if (qitem->p) {
             erts_proc_lock(qitem->p, ERTS_PROC_LOCK_STATUS);
+
             if (!ERTS_PROC_IS_EXITING(qitem->p)) {
                 erts_resume(qitem->p, ERTS_PROC_LOCK_STATUS);
             }
+
             erts_proc_unlock(qitem->p, ERTS_PROC_LOCK_STATUS);
             erts_proc_dec_refc(qitem->p);
-        }
-        else { /* aux work*/
+        } else { /* aux work */
             ErtsSchedulerData *esdp = erts_get_scheduler_data();
             ASSERT(esdp && esdp->type == ERTS_SCHED_NORMAL);
             erts_schedule_misc_aux_work((int) esdp->no,
                                         qitem->aux_func,
                                         qitem->aux_arg);
         }
-        code_write_queue = qitem->next;
-	erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem);
+
+        perm->queue = qitem->next;
+        erts_free(ERTS_ALC_T_CODE_IX_LOCK_Q, qitem);
+    }
+
+    perm->scheduler = NULL;
+    perm->owner = NULL;
+    perm->seized = 0;
+
+    erts_mtx_unlock(&perm->lock);
+}
+
+int erts_try_seize_code_mod_permission_aux(void (*aux_func)(void *),
+                                           void *aux_arg)
+{
+    ASSERT(aux_func != NULL);
+    return try_seize_code_permission(&code_mod_permission, NULL,
+                                     aux_func, aux_arg);
+}
+
+int erts_try_seize_code_mod_permission(Process* c_p)
+{
+    ASSERT(c_p != NULL);
+
+#ifdef CWP_DBG_FORCE_TRAP
+    if (!(c_p->flags & F_DBG_FORCED_TRAP)) {
+        c_p->flags |= F_DBG_FORCED_TRAP;
+        return 0;
+    } else {
+        /* back from forced trap */
+        c_p->flags &= ~F_DBG_FORCED_TRAP;
+    }
+#endif
+
+    return try_seize_code_permission(&code_mod_permission, c_p, NULL, NULL);
+}
+
+void erts_release_code_mod_permission(void)
+{
+    release_code_permission(&code_mod_permission);
+}
+
+int erts_try_seize_code_stage_permission(Process* c_p)
+{
+    ASSERT(c_p != NULL);
+
+#ifdef CWP_DBG_FORCE_TRAP
+    if (!(c_p->flags & F_DBG_FORCED_TRAP)) {
+        c_p->flags |= F_DBG_FORCED_TRAP;
+        return 0;
+    } else {
+        /* back from forced trap */
+        c_p->flags &= ~F_DBG_FORCED_TRAP;
+    }
+#endif
+
+    return try_seize_code_permission(&code_stage_permission, c_p, NULL, NULL);
+}
+
+void erts_release_code_stage_permission(void) {
+    release_code_permission(&code_stage_permission);
+}
+
+int erts_try_seize_code_load_permission(Process* c_p) {
+    ASSERT(c_p != NULL);
+
+#ifdef CWP_DBG_FORCE_TRAP
+    if (!(c_p->flags & F_DBG_FORCED_TRAP)) {
+        c_p->flags |= F_DBG_FORCED_TRAP;
+        return 0;
+    } else {
+        /* back from forced trap */
+        c_p->flags &= ~F_DBG_FORCED_TRAP;
     }
-    code_writing_seized = 0;
-    code_writing_process = NULL;
-#ifdef ERTS_ENABLE_LOCK_CHECK
-    erts_tsd_set(has_code_write_permission, (void *) 0);
 #endif
-    erts_mtx_unlock(&code_write_permission_mtx);
+
+    if (try_seize_code_permission(&code_stage_permission, c_p, NULL, NULL)) {
+        if (try_seize_code_permission(&code_mod_permission, c_p, NULL, NULL)) {
+            return 1;
+        }
+
+        erts_release_code_stage_permission();
+    }
+
+    return 0;
+}
+
+void erts_release_code_load_permission(void) {
+    erts_release_code_mod_permission();
+    erts_release_code_stage_permission();
 }
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
-int erts_has_code_write_permission(void)
+static int has_code_permission(struct code_permission *perm)
 {
-    return code_writing_seized && erts_tsd_get(has_code_write_permission);
+    const ErtsSchedulerData *esdp = erts_get_scheduler_data();
+
+    if (esdp && esdp->type == ERTS_SCHED_NORMAL) {
+        int res;
+
+        erts_mtx_lock(&perm->lock);
+
+        res = perm->seized;
+
+        if (esdp->current_process != NULL) {
+            /* If we're running a process, it has to match the owner of the
+             * permission. We don't care about which scheduler we are running 
+             * on in order to support holding permissions when yielding (such
+             * as in code purging). */
+            res &= perm->owner == esdp->current_process;
+        } else {
+            /* If we're running an aux job, we crudely assume that this current
+             * job was started by the owner if there is one, and therefore has
+             * permission.
+             *
+             * If we don't have an owner, we assume that we have permission if
+             * we're running on the same scheduler that started the job.
+             *
+             * This is very blunt and only catches _some_ cases where we lack
+             * lack permission, but at least it's better than the old method of
+             * using thread-specific-data. */
+            res &= perm->owner || perm->scheduler == esdp;
+        }
+
+        erts_mtx_unlock(&perm->lock);
+
+        return res;
+    }
+
+    return 0;
+}
+
+int erts_has_code_load_permission() {
+    return erts_has_code_stage_permission() && erts_has_code_mod_permission();
+}
+
+int erts_has_code_stage_permission() {
+    return has_code_permission(&code_stage_permission);
+}
+
+int erts_has_code_mod_permission() {
+    return has_code_permission(&code_mod_permission);
 }
 #endif
 
@@ -278,29 +398,7 @@ static void issue_instruction_barrier(void *barrier_) {
     ERTS_THR_INSTRUCTION_BARRIER;
 
     if (erts_refc_dectest(&barrier->pending_schedulers, 0) == 0) {
-#    ifdef ERTS_ENABLE_LOCK_CHECK
-        ErtsSchedulerData *initial_esdp = (ErtsSchedulerData*)barrier->esdp;
-
-        /* HACK: Dodges a broken lock-checking assertion, which requires that
-         * the _thread_ that takes a code permission is also the one that
-         * releases it.
-         *
-         * This has never held since processes can migrate between schedulers,
-         * but we have to roll with the punches. Commit the code on the
-         * scheduler that called `erts_schedule_code_barrier` in the hopes that
-         * it's the right one.
-         *
-         * This has been fixed in `master`. */
-        if (initial_esdp && initial_esdp != erts_get_scheduler_data()) {
-            erts_schedule_misc_aux_work(initial_esdp->no,
-                                        schedule_code_barrier_later_op,
-                                        barrier);
-        } else {
-            schedule_code_barrier_later_op(barrier);
-        }
-#    else
         schedule_code_barrier_later_op(barrier);
-#    endif
     }
 }
 #endif
@@ -320,7 +418,6 @@ void erts_schedule_code_barrier_cleanup(ErtsCodeBarrier *barrier,
     erts_debug_unrequire_code_barrier();
 #endif
 
-    barrier->esdp = erts_get_scheduler_data();
     barrier->later_function = later_function;
     barrier->later_data = later_data;
     barrier->size = size;
@@ -377,7 +474,7 @@ static void schedule_blocking_code_barriers(void *ignored) {
 }
 #endif
 
-void erts_blocking_code_barrier()
+void erts_blocking_code_barrier(void)
 {
 #ifdef DEBUG
     erts_debug_unrequire_code_barrier();
@@ -390,7 +487,7 @@ void erts_blocking_code_barrier()
 #endif
 }
 
-void erts_code_ix_finalize_wait() {
+void erts_code_ix_finalize_wait(void) {
 #ifdef CODE_IX_ISSUE_INSTRUCTION_BARRIERS
     if (erts_atomic32_read_nob(&outstanding_blocking_code_barriers) != 0) {
         ERTS_THR_INSTRUCTION_BARRIER;