diff options
Diffstat (limited to 'erts/emulator/beam/instrs.tab')
| -rw-r--r-- | erts/emulator/beam/instrs.tab | 259 |
1 files changed, 173 insertions, 86 deletions
diff --git a/erts/emulator/beam/instrs.tab b/erts/emulator/beam/instrs.tab index 6d32d44d9a..697c777825 100644 --- a/erts/emulator/beam/instrs.tab +++ b/erts/emulator/beam/instrs.tab @@ -19,7 +19,12 @@ // %CopyrightEnd% // -// Stack manipulation instructions +// +// Stack manipulation instructions follow. +// +// See the comment for AH() in macros.tab for information about +// the layout of stack frames. +// allocate(NeedStack, Live) { $AH($NeedStack, 0, $Live); @@ -58,105 +63,170 @@ allocate_heap_zero(NeedStack, NeedHeap, Live) { deallocate(Deallocate) { //| -no_prefetch - SET_CP(c_p, (BeamInstr *) cp_val(*E)); E = ADD_BYTE_OFFSET(E, $Deallocate); } -deallocate_return(Deallocate) { - //| -no_next - int words_to_pop = $Deallocate; - SET_I((BeamInstr *) cp_val(*E)); - E = ADD_BYTE_OFFSET(E, words_to_pop); - CHECK_TERM(x(0)); - DispatchReturn; +// +// Micro-benchmarks showed that the deallocate_return instruction +// became slower when the continuation pointer was moved from +// the process struct to the stack. The reason seems to be read +// dependencies, i.e. that the CPU cannot figure out beforehand +// from which position on the stack the continuation pointer +// should be fetched. +// +// Initializing num_bytes with a constant value seems to restore +// the lost speed, so we've specialized the instruction for the +// most common values. +// + +deallocate_return0 := dealloc_ret.n0.execute; +deallocate_return1 := dealloc_ret.n1.execute; +deallocate_return2 := dealloc_ret.n2.execute; +deallocate_return3 := dealloc_ret.n3.execute; +deallocate_return4 := dealloc_ret.n4.execute; +deallocate_return := dealloc_ret.var.execute; + +dealloc_ret.head() { + Uint num_bytes; } -move_deallocate_return(Src, Deallocate) { - x(0) = $Src; - $deallocate_return($Deallocate); +dealloc_ret.n0() { + num_bytes = (0+1) * sizeof(Eterm); } -// Call instructions +dealloc_ret.n1() { + num_bytes = (1+1) * sizeof(Eterm); +} + +dealloc_ret.n2() { + num_bytes = (2+1) * sizeof(Eterm); +} + +dealloc_ret.n3() { + num_bytes = (3+1) * sizeof(Eterm); +} -DISPATCH_REL(CallDest) { +dealloc_ret.n4() { + num_bytes = (4+1) * sizeof(Eterm); +} + +dealloc_ret.var(Deallocate) { + num_bytes = $Deallocate; +} + +dealloc_ret.execute() { //| -no_next - $SET_I_REL($CallDest); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); + + E = ADD_BYTE_OFFSET(E, num_bytes); + $RETURN(); + CHECK_TERM(x(0)); + $DISPATCH_RETURN(); } -DISPATCH_ABS(CallDest) { +move_deallocate_return(Src, Deallocate) { //| -no_next - SET_I((BeamInstr *) $CallDest); - DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); - Dispatch(); + + /* + * Explicitly do reads first to mitigate the impact of read + * dependencies. + */ + + Uint bytes_to_pop = $Deallocate; + Eterm src = $Src; + E = ADD_BYTE_OFFSET(E, bytes_to_pop); + x(0) = src; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); + CHECK_TERM(x(0)); + $DISPATCH_RETURN(); } +// Call instructions + i_call(CallDest) { - SET_CP(c_p, $NEXT_INSTRUCTION); + //| -no_next + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_REL($CallDest); } move_call(Src, CallDest) { - x(0) = $Src; - SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCH_REL($CallDest); + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_last(CallDest, Deallocate) { + //| -no_next $deallocate($Deallocate); $DISPATCH_REL($CallDest); } move_call_last(Src, CallDest, Deallocate) { - x(0) = $Src; - $i_call_last($CallDest, $Deallocate); + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $deallocate($Deallocate); + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_only(CallDest) { + //| -no_next $DISPATCH_REL($CallDest); } move_call_only(Src, CallDest) { - x(0) = $Src; - $i_call_only($CallDest); -} - -DISPATCHX(Dest) { //| -no_next - DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, $Dest); - // Dispatchx assumes the Export* is in Arg(0) - I = (&$Dest) - 1; - Dispatchx(); + Eterm call_dest = $CallDest; + Eterm src = $Src; + x(0) = src; + $DISPATCH_REL(call_dest); } i_call_ext(Dest) { - SET_CP(c_p, $NEXT_INSTRUCTION); - $DISPATCHX($Dest); + //| -no_next + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + $DISPATCH_EXPORT($Dest); } -i_move_call_ext(Src, Dest) { - x(0) = $Src; - $i_call_ext($Dest); +i_move_call_ext(Src, CallDest) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); + x(0) = src; + $DISPATCH_EXPORT(call_dest); } i_call_ext_only(Dest) { - $DISPATCHX($Dest); + //| -no_next + $DISPATCH_EXPORT($Dest); } -i_move_call_ext_only(Dest, Src) { - x(0) = $Src; - $i_call_ext_only($Dest); +i_move_call_ext_only(CallDest, Src) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + x(0) = src; + $DISPATCH_EXPORT(call_dest); } i_call_ext_last(Dest, Deallocate) { + //| -no_next $deallocate($Deallocate); - $DISPATCHX($Dest); + $DISPATCH_EXPORT($Dest); } -i_move_call_ext_last(Dest, StackOffset, Src) { - x(0) = $Src; - $i_call_ext_last($Dest, $StackOffset); +i_move_call_ext_last(CallDest, Deallocate, Src) { + //| -no_next + Eterm call_dest = $CallDest; + Eterm src = $Src; + $deallocate($Deallocate); + x(0) = src; + $DISPATCH_EXPORT(call_dest); } APPLY(I, Deallocate, Next) { @@ -167,21 +237,24 @@ APPLY(I, Deallocate, Next) { } HANDLE_APPLY_ERROR() { - I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); + static ErtsCodeMFA apply3_mfa = {am_erlang, am_apply, 3}; + I = handle_error(c_p, I, reg, &apply3_mfa); goto post_error_handling; } i_apply() { + //| -no_next BeamInstr *next; $APPLY(NULL, 0, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } i_apply_last(Deallocate) { + //| -no_next BeamInstr *next; $APPLY(I, $Deallocate, next); if (ERTS_LIKELY(next != NULL)) { @@ -192,6 +265,7 @@ i_apply_last(Deallocate) { } i_apply_only() { + //| -no_next BeamInstr *next; $APPLY(I, 0, next); if (ERTS_LIKELY(next != NULL)) { @@ -208,16 +282,18 @@ FIXED_APPLY(Arity, I, Deallocate, Next) { } apply(Arity) { + //| -no_next BeamInstr *next; $FIXED_APPLY($Arity, NULL, 0, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_ABS(next); } $HANDLE_APPLY_ERROR(); } apply_last(Arity, Deallocate) { + //| -no_next BeamInstr *next; $FIXED_APPLY($Arity, I, $Deallocate, next); if (ERTS_LIKELY(next != NULL)) { @@ -237,23 +313,19 @@ HANDLE_APPLY_FUN_ERROR() { goto find_func_info; } -DISPATCH_FUN(I) { - //| -no_next - SET_I($I); - Dispatchfun(); -} - i_apply_fun() { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); } i_apply_fun_last(Deallocate) { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { @@ -264,6 +336,7 @@ i_apply_fun_last(Deallocate) { } i_apply_fun_only() { + //| -no_next BeamInstr *next; $APPLY_FUN(next); if (ERTS_LIKELY(next != NULL)) { @@ -280,16 +353,18 @@ CALL_FUN(Fun, Next) { } i_call_fun(Fun) { + //| -no_next BeamInstr *next; $CALL_FUN($Fun, next); if (ERTS_LIKELY(next != NULL)) { - SET_CP(c_p, $NEXT_INSTRUCTION); + $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION); $DISPATCH_FUN(next); } $HANDLE_APPLY_FUN_ERROR(); } i_call_fun_last(Fun, Deallocate) { + //| -no_next BeamInstr *next; $CALL_FUN($Fun, next); if (ERTS_LIKELY(next != NULL)) { @@ -301,18 +376,12 @@ i_call_fun_last(Fun, Deallocate) { return() { //| -no_next - SET_I(c_p->cp); - DTRACE_RETURN_FROM_PC(c_p); - - /* - * We must clear the CP to make sure that a stale value do not - * create a false module dependcy preventing code upgrading. - * It also means that we can use the CP in stack backtraces. - */ - c_p->cp = 0; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); CHECK_TERM(r(0)); HEAP_SPACE_VERIFIED(0); - DispatchReturn; + + $DISPATCH_RETURN(); } get_list(Src, Hd, Tl) { @@ -478,16 +547,21 @@ i_make_fun(FunP, NumFree) { } move_trim(Src, Dst, Words) { - Uint cp = E[0]; $Dst = $Src; - E += $Words; - E[0] = cp; + $i_trim($Words); } i_trim(Words) { - Uint cp = E[0]; E += $Words; - E[0] = cp; + + /* + * Clear the reserved location for the continuation pointer at + * E[0]. This is not strictly necessary for correctness, but if a + * GC is triggered before E[0] is overwritten by another + * continuation pointer the now dead term at E[0] would be + * retained by the GC. + */ + E[0] = NIL; } move(Src, Dst) { @@ -599,9 +673,9 @@ move_window5(S1, S2, S3, S4, S5, D) { move_return(Src) { //| -no_next x(0) = $Src; - SET_I(c_p->cp); - c_p->cp = 0; - DispatchReturn; + DTRACE_RETURN_FROM_PC(c_p, I); + $RETURN(); + $DISPATCH_RETURN(); } move_x1(Src) { @@ -683,10 +757,11 @@ swap(R1, R2) { $R2 = V; } -swap_temp(R1, R2, Tmp) { - Eterm V = $R1; - $R1 = $R2; - $R2 = $Tmp = V; +swap2(R1, R2, R3) { + Eterm V = $R2; + $R2 = $R1; + $R1 = $R3; + $R3 = V; } test_heap(Nh, Live) { @@ -952,8 +1027,17 @@ is_ge(Fail, X, Y) { CMP_GE_ACTION($X, $Y, $FAIL($Fail)); } -badarg(Fail) { - $BADARG($Fail); +is_lt_literal(Fail, X, Y) { + CMP_LT_LITERAL_ACTION($X, $Y, $FAIL($Fail)); +} + +is_ge_literal(Fail, X, Y) { + CMP_GE_LITERAL_ACTION($X, $Y, $FAIL($Fail)); +} + +badarg_body() { + c_p->freason = BADARG; + $FAIL_BODY(); //| -no_next; } @@ -977,8 +1061,9 @@ if_end() { //| -no_next; } -system_limit(Fail) { - $SYSTEM_LIMIT($Fail); +system_limit_body() { + c_p->freason = SYSTEM_LIMIT; + $FAIL_BODY(); //| -no_next; } @@ -991,6 +1076,7 @@ catch_end(Y) { $try_end($Y); if (is_non_value(r(0))) { c_p->fvalue = NIL; + c_p->ftrace = NIL; if (x(1) == am_throw) { r(0) = x(2); } else { @@ -1024,6 +1110,7 @@ try_case(Y) { $try_end($Y); ASSERT(is_non_value(r(0))); c_p->fvalue = NIL; + c_p->ftrace = NIL; r(0) = x(1); x(1) = x(2); x(2) = x(3); |