diff options
-rw-r--r-- | Zend/zend_execute.c | 5 | ||||
-rw-r--r-- | Zend/zend_vm_execute.h | 33 | ||||
-rwxr-xr-x | Zend/zend_vm_gen.php | 39 | ||||
-rw-r--r-- | Zend/zend_vm_opcodes.h | 6 | ||||
-rw-r--r-- | ext/opcache/jit/zend_jit_x86.dasc | 52 |
5 files changed, 105 insertions, 30 deletions
diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c index c58e1fa228..fb35b8a116 100644 --- a/Zend/zend_execute.c +++ b/Zend/zend_execute.c @@ -4757,6 +4757,11 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint } \ } while (0) +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE +/* This callback disables optimization of "vm_stack_data" variable in VM */ +void (*zend_touch_vm_stack_data)(void *vm_stack_data) = NULL; +#endif + #include "zend_vm_execute.h" ZEND_API zend_result zend_set_user_opcode_handler(zend_uchar opcode, user_opcode_handler_t handler) diff --git a/Zend/zend_vm_execute.h b/Zend/zend_vm_execute.h index 0d14ff3bab..60d725b365 100644 --- a/Zend/zend_vm_execute.h +++ b/Zend/zend_vm_execute.h @@ -50826,15 +50826,34 @@ ZEND_API void execute_ex(zend_execute_data *ex) { DCL_OPLINE +#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG) + struct { #ifdef ZEND_VM_IP_GLOBAL_REG - const zend_op *orig_opline = opline; + const zend_op *orig_opline; #endif #ifdef ZEND_VM_FP_GLOBAL_REG - zend_execute_data *orig_execute_data = execute_data; + zend_execute_data *orig_execute_data; +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + char hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE]; +#endif +#endif + } vm_stack_data; +#endif +#ifdef ZEND_VM_IP_GLOBAL_REG + vm_stack_data.orig_opline = opline; +#endif +#ifdef ZEND_VM_FP_GLOBAL_REG + vm_stack_data.orig_execute_data = execute_data; execute_data = ex; #else zend_execute_data *execute_data = ex; #endif +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE); + if (zend_touch_vm_stack_data) { + zend_touch_vm_stack_data(&vm_stack_data); + } +#endif #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) if (UNEXPECTED(execute_data == NULL)) { @@ -58788,10 +58807,10 @@ zend_leave_helper_SPEC_LABEL: HYBRID_BREAK(); HYBRID_CASE(HYBRID_HALT): #ifdef ZEND_VM_FP_GLOBAL_REG - execute_data = orig_execute_data; + execute_data = vm_stack_data.orig_execute_data; #endif #ifdef ZEND_VM_IP_GLOBAL_REG - opline = orig_opline; + opline = vm_stack_data.orig_opline; #endif return; HYBRID_DEFAULT: @@ -58800,9 +58819,9 @@ zend_leave_helper_SPEC_LABEL: HYBRID_BREAK(); /* Never reached */ #else #ifdef ZEND_VM_FP_GLOBAL_REG - execute_data = orig_execute_data; + execute_data = vm_stack_data.orig_execute_data; # ifdef ZEND_VM_IP_GLOBAL_REG - opline = orig_opline; + opline = vm_stack_data.orig_opline; # endif return; #else @@ -58811,7 +58830,7 @@ zend_leave_helper_SPEC_LABEL: ZEND_VM_LOOP_INTERRUPT_CHECK(); } else { # ifdef ZEND_VM_IP_GLOBAL_REG - opline = orig_opline; + opline = vm_stack_data.orig_opline; # endif return; } diff --git a/Zend/zend_vm_gen.php b/Zend/zend_vm_gen.php index 4958189ce4..46030cad09 100755 --- a/Zend/zend_vm_gen.php +++ b/Zend/zend_vm_gen.php @@ -1780,10 +1780,10 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array()) case ZEND_VM_KIND_HYBRID: out($f,"\t\t\tHYBRID_CASE(HYBRID_HALT):\n"); out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n"); - out($f,"\t\t\t\texecute_data = orig_execute_data;\n"); + out($f,"\t\t\t\texecute_data = vm_stack_data.orig_execute_data;\n"); out($f,"#endif\n"); out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n"); - out($f,"\t\t\t\topline = orig_opline;\n"); + out($f,"\t\t\t\topline = vm_stack_data.orig_opline;\n"); out($f,"#endif\n"); out($f,"\t\t\t\treturn;\n"); out($f,"\t\t\tHYBRID_DEFAULT:\n"); @@ -2065,15 +2065,34 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) out($f,$m[1]."zend_execute_data *execute_data = ex;\n"); out($f,"#endif\n"); } else { + out($f,"#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)\n"); + out($f,$m[1]."struct {\n"); out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n"); - out($f,$m[1]."const zend_op *orig_opline = opline;\n"); + out($f,$m[1]."\tconst zend_op *orig_opline;\n"); out($f,"#endif\n"); out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n"); - out($f,$m[1]."zend_execute_data *orig_execute_data = execute_data;\n"); + out($f,$m[1]."\tzend_execute_data *orig_execute_data;\n"); + out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n"); + out($f,$m[1]."\tchar hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];\n"); + out($f,"#endif\n"); + out($f,"#endif\n"); + out($f,$m[1]."} vm_stack_data;\n"); + out($f,"#endif\n"); + out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n"); + out($f,$m[1]."vm_stack_data.orig_opline = opline;\n"); + out($f,"#endif\n"); + out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n"); + out($f,$m[1]."vm_stack_data.orig_execute_data = execute_data;\n"); out($f,$m[1]."execute_data = ex;\n"); out($f,"#else\n"); out($f,$m[1]."zend_execute_data *execute_data = ex;\n"); out($f,"#endif\n"); + out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n"); + out($f,$m[1]."memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);\n"); + out($f,$m[1]."if (zend_touch_vm_stack_data) {\n"); + out($f,$m[1]."\tzend_touch_vm_stack_data(&vm_stack_data);\n"); + out($f,$m[1]."}\n"); + out($f,"#endif\n"); } break; case "INTERNAL_LABELS": @@ -2159,9 +2178,9 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) } out($f, "#ifdef ZEND_VM_FP_GLOBAL_REG\n" . - $m[1]."execute_data = orig_execute_data;\n" . + $m[1]."execute_data = vm_stack_data.orig_execute_data;\n" . "# ifdef ZEND_VM_IP_GLOBAL_REG\n" . - $m[1]."opline = orig_opline;\n" . + $m[1]."opline = vm_stack_data.orig_opline;\n" . "# endif\n" . $m[1]."return;\n" . "#else\n" . @@ -2170,7 +2189,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name) $m[1]."\tZEND_VM_LOOP_INTERRUPT_CHECK();\n". $m[1]."} else {\n" . "# ifdef ZEND_VM_IP_GLOBAL_REG\n" . - $m[1]."\topline = orig_opline;\n" . + $m[1]."\topline = vm_stack_data.orig_opline;\n" . "# endif\n". $m[1]."\treturn;\n". $m[1]."}\n". @@ -2578,6 +2597,12 @@ function gen_vm($def, $skel) { fputs($f, "#define ZEND_VM_KIND\t\t" . $GLOBALS["vm_kind_name"][ZEND_VM_KIND] . "\n"); } fputs($f, "\n"); + fputs($f, "#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)\n"); + fputs($f, "# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))\n"); + fputs($f, "# define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16\n"); + fputs($f, "# endif\n"); + fputs($f, "#endif\n"); + fputs($f, "\n"); foreach($vm_op_flags as $name => $val) { fprintf($f, "#define %-24s 0x%08x\n", $name, $val); } diff --git a/Zend/zend_vm_opcodes.h b/Zend/zend_vm_opcodes.h index df5a147999..584a44f5ae 100644 --- a/Zend/zend_vm_opcodes.h +++ b/Zend/zend_vm_opcodes.h @@ -34,6 +34,12 @@ # define ZEND_VM_KIND ZEND_VM_KIND_CALL #endif +#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__) +# if (defined(i386) || defined(__x86_64__) || defined(_M_X64)) +# define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16 +# endif +#endif + #define ZEND_VM_OP_SPEC 0x00000001 #define ZEND_VM_OP_CONST 0x00000002 #define ZEND_VM_OP_TMPVAR 0x00000004 diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index ca06f7bdea..9d518ae9e5 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -171,8 +171,21 @@ static void* dasm_labels[zend_lb_MAX]; #define BP_JIT_IS 6 + #define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) +|.macro ADD_HYBRID_SPAD +||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE +| add r4, HYBRID_SPAD +||#endif +|.endmacro + +|.macro SUB_HYBRID_SPAD +||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE +| sub r4, HYBRID_SPAD +||#endif +|.endmacro + |.macro LOAD_ADDR, reg, addr | .if X64 || if (IS_32BIT(addr)) { @@ -1752,7 +1765,7 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) } | //ZEND_VM_CONTINUE() if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | JMP_IP } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment @@ -1774,7 +1787,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { const void *handler = zend_get_opcode_handler_func(EG(exception_op)); - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | EXT_CALL handler, r0 | JMP_IP } else { @@ -1827,11 +1840,11 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | test FCARG1d, ZEND_CALL_TOP | jnz >1 | EXT_CALL zend_jit_leave_nested_func_helper, r0 - | add r4, HYBRID_SPAD // stack alignment + | ADD_HYBRID_SPAD | JMP_IP |1: | EXT_CALL zend_jit_leave_top_func_helper, r0 - | add r4, HYBRID_SPAD // stack alignment + | ADD_HYBRID_SPAD | JMP_IP } else { if (GCC_GLOBAL_REGS) { @@ -2413,7 +2426,7 @@ static int zend_jit_trace_halt_stub(dasm_State **Dst) { |->trace_halt: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | EXT_JMP zend_jit_halt_op->handler, r0 } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment @@ -2505,7 +2518,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | LOAD_IP if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | JMP_IP } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment @@ -2531,7 +2544,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | jne ->interrupt_handler if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] @@ -2566,7 +2579,7 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst) |->trace_escape: | if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | JMP_IP } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment @@ -2608,7 +2621,7 @@ static int zend_jit_context_threaded_call_stub(dasm_State **Dst) |->context_threaded_call: | pop r0 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | jmp aword [IP] } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment @@ -2990,7 +3003,7 @@ static int zend_jit_align_func(dasm_State **Dst) static int zend_jit_prologue(dasm_State **Dst) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | sub r4, HYBRID_SPAD + | SUB_HYBRID_SPAD } else if (GCC_GLOBAL_REGS) { | sub r4, SPAD // stack alignment } else { @@ -3344,10 +3357,13 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, /* Skip prologue. */ // TODO: don't hardcode this ??? if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + prologue_size = 0; +#elif defined(__x86_64__) || defined(_M_X64) // sub r4, HYBRID_SPAD -#if defined(__x86_64__) || defined(_M_X64) prologue_size = 4; #else + // sub r4, HYBRID_SPAD prologue_size = 3; #endif } else if (GCC_GLOBAL_REGS) { @@ -3387,7 +3403,7 @@ static int zend_jit_trace_return(dasm_State **Dst, zend_bool original_handler) | jmp ->trace_escape #else if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD if (!original_handler) { | JMP_IP } else { @@ -3652,13 +3668,13 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) /* Use inlined HYBRID VM handler */ const void *handler = opline->handler; - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | EXT_JMP handler, r0 } else { const void *handler = zend_get_opcode_handler_func(opline); | EXT_CALL handler, r0 - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | JMP_IP } } else { @@ -10044,7 +10060,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } #else if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD | JMP_IP } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment @@ -11283,7 +11299,11 @@ static int zend_jit_leave_func(dasm_State **Dst, trace_info->flags |= ZEND_JIT_TRACE_LOOP; | CMP_IP next_opline | je =>0 // LOOP +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + | JMP_IP +#else | jmp ->trace_escape +#endif } else { | CMP_IP next_opline | jne ->trace_escape @@ -11313,7 +11333,7 @@ static int zend_jit_leave_func(dasm_State **Dst, } if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | add r4, HYBRID_SPAD + | ADD_HYBRID_SPAD #ifdef CONTEXT_THREADED_JIT | push aword [IP] | ret |