diff options
| -rw-r--r-- | ext/opcache/jit/zend_jit_x86.dasc | 603 |
1 files changed, 201 insertions, 402 deletions
diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 49a25343da..85af2ad12c 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -42,7 +42,6 @@ |.define FCARG2d, CARG2d |.define SPAD, 0x08 // padding for CPU stack alignment |.define NR_SPAD, 0x58 // padding for CPU stack alignment - |.define SSE, 1 |.define T3, [r4+0x50] // Used to store old value of IP |.define T2, [r4+0x48] // Used to store old value of FP |.define T1, [r4+0x40] @@ -71,7 +70,6 @@ |.define FCARG2d, CARG2d |.define SPAD, 0x08 // padding for CPU stack alignment |.define NR_SPAD, 0x18 // padding for CPU stack alignment - |.define SSE, 1 |.define T3, [r4+0x10] // Used to store old value of IP (CALL VM only) |.define T2, [r4+0x08] // Used to store old value of FP (CALL VM only) |.define T1, [r4] @@ -86,7 +84,6 @@ |.define FCARG2d, edx |.define SPAD, 12 // padding for CPU stack alignment |.define NR_SPAD, 12 // padding for CPU stack alignment - |.define SSE, 1 |.define T3, [r4+0x10] // Used to store old value of IP (CALL VM only) |.define T2, [r4+0x08] // Used to store old value of FP (CALL VM only) |.define T1, [r4] @@ -516,42 +513,6 @@ static void* dasm_labels[zend_lb_MAX]; | SET_Z_TYPE_INFO FP + r0, IS_UNDEF |.endmacro -|.macro FPU_OP, fp_ins, addr -|| if (Z_MODE(addr) == IS_CONST_ZVAL) { -| MEM_OP1 fp_ins, qword, Z_ZV(addr), r0 -|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| fp_ins qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] -|| } else { -|| ZEND_ASSERT(0); -|| } -|.endmacro - -|.macro FPU_GET_ZVAL_DVAL, addr -| FPU_OP fld, addr -|.endmacro - -|.macro FPU_MATH, opcode, addr -|| switch (opcode) { -|| case ZEND_ADD: -| FPU_OP fadd, addr -|| break; -|| case ZEND_SUB: -| FPU_OP fsub, addr -|| break; -|| case ZEND_MUL: -| FPU_OP fmul, addr -|| break; -|| case ZEND_DIV: -| FPU_OP fdiv, addr -|| break; -|| } -|.endmacro - -|.macro FPU_SET_ZVAL_DVAL, addr -|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| fstp qword [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)] -|.endmacro - |.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 || if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { | avx_ins op1, op2 @@ -887,69 +848,25 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro FPU_LONG_OP, fp_ins, addr -|| if (Z_MODE(addr) == IS_CONST_ZVAL) { -| MEM_OP1 fp_ins, aword, Z_ZV(addr), r0 -|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| fp_ins aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] -|| } else if (Z_MODE(addr) == IS_REG) { -| fp_ins Ra(Z_REG(addr)) -|| } else { -|| ZEND_ASSERT(0); -|| } -|.endmacro - -|.macro FPU_GET_ZVAL_LVAL, addr -| FPU_LONG_OP fild, addr -|.endmacro - -|.macro FPU_MATH_REG, opcode, reg -|| switch (opcode) { -|| case ZEND_ADD: -| fadd reg -|| break; -|| case ZEND_SUB: -| fsub reg -|| break; -|| case ZEND_MUL: -| fmul reg -|| break; -|| case ZEND_DIV: -| fdiv reg -|| break; -|| } -|.endmacro - |.macro ZVAL_COPY_CONST, dst_addr, dst_info, zv, tmp_reg || if (Z_TYPE_P(zv) > IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; -| .if X64 or SSE -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { -| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) -|| } else { -| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) -|| } -| .if X64 -|| } else if (!IS_32BIT(zv)) { -| mov64 tmp_reg, ((uintptr_t)zv) -| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] -| .endif +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } else { -| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] -|| } -| SSE_SET_ZVAL_DVAL dst_addr, dst_reg -| .else -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| fldz -|| } else if (Z_DVAL_P(zv) == 1.0) { -| fld1 -|| } else { -| fld qword [zv] +| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } -| FPU_SET_ZVAL_DVAL dst_addr -| .endif +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] +| .endif +|| } else { +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] +|| } +| SSE_SET_ZVAL_DVAL dst_addr, dst_reg || } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { | xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) || } else { @@ -981,41 +898,22 @@ static void* dasm_labels[zend_lb_MAX]; || if (Z_TYPE_P(zv) == IS_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? || Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); -| .if X64 or SSE -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { -| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) -|| } else { -| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) -|| } -| .if X64 -|| } else if (!IS_32BIT(zv)) { -| mov64 tmp_reg, ((uintptr_t)zv) -| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] -| .endif +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } else { -| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] -|| } -| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 -| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 -| .else -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| fldz -|| } else if (Z_DVAL_P(zv) == 1.0) { -| fld1 -|| } else { -| fld qword [zv] -|| } -| FPU_SET_ZVAL_DVAL dst_addr -|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { -| fldz -|| } else if (Z_DVAL_P(zv) == 1.0) { -| fld1 -|| } else { -| fld qword [zv] +| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } -| FPU_SET_ZVAL_DVAL res_addr -| .endif +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] +| .endif +|| } else { +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] +|| } +| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 || } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { || if (Z_MODE(dst_addr) == IS_REG) { | xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) @@ -1087,19 +985,14 @@ static void* dasm_labels[zend_lb_MAX]; | SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) || } || } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { -| .if X64 or SSE -|| if (Z_MODE(src_addr) == IS_REG) { -| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) -|| } else if (Z_MODE(dst_addr) == IS_REG) { -| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr -|| } else { -| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr -| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 -|| } -| .else -| FPU_GET_ZVAL_DVAL src_addr -| FPU_SET_ZVAL_DVAL dst_addr -| .endif +|| if (Z_MODE(src_addr) == IS_REG) { +| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) +|| } else if (Z_MODE(dst_addr) == IS_REG) { +| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr +|| } else { +| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr +| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +|| } || } else if (!(src_info & MAY_BE_DOUBLE)) { | GET_ZVAL_PTR Ra(tmp_reg2), src_addr | SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) @@ -1160,27 +1053,20 @@ static void* dasm_labels[zend_lb_MAX]; | SET_ZVAL_LVAL res_addr, Ra(tmp_reg2) || } || } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { -| .if X64 or SSE -|| if (Z_MODE(src_addr) == IS_REG) { -| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) -| SSE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) -|| } else if (Z_MODE(dst_addr) == IS_REG) { -| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr -| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) -|| } else if (Z_MODE(res_addr) == IS_REG) { -| SSE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr -| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) -|| } else { -| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr -| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 -| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 -|| } -| .else -| FPU_GET_ZVAL_DVAL src_addr -| FPU_STROE dst_addr -| FPU_GET_ZVAL_DVAL src_addr -| FPU_SET_ZVAL_DVAL res_addr -| .endif +|| if (Z_MODE(src_addr) == IS_REG) { +| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) +| SSE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) +|| } else if (Z_MODE(dst_addr) == IS_REG) { +| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr +| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| SSE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr +| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) +|| } else { +| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr +| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 +|| } || } else if (!(src_info & MAY_BE_DOUBLE)) { | GET_ZVAL_PTR Ra(tmp_reg2), src_addr | SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) @@ -2490,17 +2376,15 @@ extern char *_tls_end; static int zend_jit_setup(void) { - |.if SSE - if (!zend_cpu_supports(ZEND_CPU_FEATURE_SSE2)) { - zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); - return FAILURE; - } - if (zend_jit_cpu_flags & ZEND_JIT_CPU_AVX) { - if (zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) { - zend_jit_x86_flags |= ZEND_JIT_CPU_AVX; - } + if (!zend_cpu_supports(ZEND_CPU_FEATURE_SSE2)) { + zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); + return FAILURE; + } + if (zend_jit_cpu_flags & ZEND_JIT_CPU_AVX) { + if (zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) { + zend_jit_x86_flags |= ZEND_JIT_CPU_AVX; } - |.endif + } #if ZTS # ifdef _WIN64 @@ -2881,11 +2765,7 @@ static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_ad | SET_ZVAL_TYPE_INFO dst, IS_LONG } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | .if SSE - | SSE_SET_ZVAL_DVAL dst, Z_REG(src) - | .else - || ZEND_ASSERT(0); - | .endif + | SSE_SET_ZVAL_DVAL dst, Z_REG(src) if (set_type) { | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE } @@ -2903,11 +2783,7 @@ static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | GET_ZVAL_LVAL Z_REG(dst), src } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | .if SSE - | SSE_GET_ZVAL_DVAL Z_REG(dst), src - | .else - || ZEND_ASSERT(0); - | .endif + | SSE_GET_ZVAL_DVAL Z_REG(dst), src } else { ZEND_ASSERT(0); } @@ -2961,11 +2837,7 @@ static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_ad if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | .if SSE - | SSE_AVX_INS movsd, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) - | .else - || ZEND_ASSERT(0); - | .endif + | SSE_AVX_INS movsd, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) } else { ZEND_ASSERT(0); } @@ -3151,39 +3023,28 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_arr | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 } } - |.if SSE - if (Z_MODE(op1_def_addr) == IS_REG) { - tmp_reg = Z_REG(op1_def_addr); - } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { - tmp_reg = Z_REG(op1_addr); - } else { - tmp_reg = ZREG_XMM0; - } - | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr - if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] - } else { - | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] - } + if (Z_MODE(op1_def_addr) == IS_REG) { + tmp_reg = Z_REG(op1_def_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + tmp_reg = Z_REG(op1_addr); + } else { + tmp_reg = ZREG_XMM0; + } + | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] } else { - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] - } else { - | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] - } + | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] } - | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg - |.else - | FPU_GET_ZVAL_DVAL op1_addr - | fld1 - if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | fadd + } else { + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] } else { - | fsub + | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] } - | FPU_SET_ZVAL_DVAL op1_def_addr - |.endif + } + | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg } if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { @@ -3262,24 +3123,17 @@ static int zend_jit_math_long_long(dasm_State **Dst, | jo >1 |.cold_code |1: - |.if SSE - zend_reg tmp_reg1 = ZREG_XMM0; - zend_reg tmp_reg2 = ZREG_XMM1; - - | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr - | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 - } else { - | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 - } - | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 - |.else - | FPU_GET_ZVAL_LVAL op2_addr - | FPU_GET_ZVAL_LVAL op1_addr - | FPU_MATH_REG opcode, st1 - | FPU_SET_ZVAL_DVAL res_addr - |.endif + zend_reg tmp_reg1 = ZREG_XMM0; + zend_reg tmp_reg2 = ZREG_XMM1; + + | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 + } else { + | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 + } + | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } @@ -3314,22 +3168,16 @@ static int zend_jit_math_long_double(dasm_State **Dst, zend_jit_addr res_addr, uint32_t res_use_info) { - |.if SSE - zend_reg result_reg = - (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; + zend_reg result_reg = + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; - | SSE_GET_ZVAL_LVAL result_reg, op1_addr - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - | AVX_MATH opcode, result_reg, result_reg, op2_addr - } else { - | SSE_MATH opcode, result_reg, op2_addr - } - | SSE_SET_ZVAL_DVAL res_addr, result_reg - |.else - | FPU_GET_ZVAL_LVAL op1_addr - | FPU_MATH opcode, op2_addr - | FPU_SET_ZVAL_DVAL res_addr - |.endif + | SSE_GET_ZVAL_LVAL result_reg, op1_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH opcode, result_reg, result_reg, op2_addr + } else { + | SSE_MATH opcode, result_reg, op2_addr + } + | SSE_SET_ZVAL_DVAL res_addr, result_reg if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { @@ -3350,62 +3198,51 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_jit_addr res_addr, uint32_t res_use_info) { - |.if SSE - zend_reg result_reg; + zend_reg result_reg; - if (zend_is_commutative(opcode)) { - if (Z_MODE(res_addr) == IS_REG) { - result_reg = Z_REG(res_addr); - } else { - result_reg = ZREG_XMM0; - } - | SSE_GET_ZVAL_LVAL result_reg, op2_addr - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - | AVX_MATH opcode, result_reg, result_reg, op1_addr - } else { - | SSE_MATH opcode, result_reg, op1_addr - } + if (zend_is_commutative(opcode)) { + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); } else { - zend_reg tmp_reg; + result_reg = ZREG_XMM0; + } + | SSE_GET_ZVAL_LVAL result_reg, op2_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH opcode, result_reg, result_reg, op1_addr + } else { + | SSE_MATH opcode, result_reg, op1_addr + } + } else { + zend_reg tmp_reg; - if (Z_MODE(res_addr) == IS_REG) { - result_reg = Z_REG(res_addr); - tmp_reg = ZREG_XMM0; - } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { - result_reg = Z_REG(op1_addr); - tmp_reg = ZREG_XMM0; - } else { - result_reg = ZREG_XMM0; - tmp_reg = ZREG_XMM1; - } - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - zend_reg op1_reg; + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + tmp_reg = ZREG_XMM0; + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + tmp_reg = ZREG_XMM0; + } else { + result_reg = ZREG_XMM0; + tmp_reg = ZREG_XMM1; + } + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + zend_reg op1_reg; - if (Z_MODE(op1_addr) == IS_REG) { - op1_reg = Z_REG(op1_addr); - } else { - | SSE_GET_ZVAL_DVAL result_reg, op1_addr - op1_reg = result_reg; - } - | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr - | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); } else { | SSE_GET_ZVAL_DVAL result_reg, op1_addr - | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr - | SSE_MATH_REG opcode, result_reg, tmp_reg + op1_reg = result_reg; } - } - | SSE_SET_ZVAL_DVAL res_addr, result_reg - |.else - | FPU_GET_ZVAL_LVAL op2_addr - if (zend_is_commutative(opcode)) { - | FPU_MATH opcode, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg } else { - | FPU_GET_ZVAL_DVAL op1_addr - | FPU_MATH_REG opcode, st1 + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | SSE_MATH_REG opcode, result_reg, tmp_reg } - | FPU_SET_ZVAL_DVAL res_addr - |.endif + } + | SSE_SET_ZVAL_DVAL res_addr, result_reg if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { @@ -3429,64 +3266,57 @@ static int zend_jit_math_double_double(dasm_State **Dst, uint32_t res_use_info) { zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; - |.if SSE - zend_reg result_reg; + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_XMM0; + } - if (Z_MODE(res_addr) == IS_REG) { - result_reg = Z_REG(res_addr); - } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { - result_reg = Z_REG(op1_addr); + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + zend_reg op1_reg; + zend_jit_addr val_addr; + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + val_addr = op2_addr; + } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { + op1_reg = Z_REG(op2_addr); + val_addr = op1_addr; } else { - result_reg = ZREG_XMM0; + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + op1_reg = result_reg; + val_addr = op2_addr; } - - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - zend_reg op1_reg; - zend_jit_addr val_addr; - - if (Z_MODE(op1_addr) == IS_REG) { - op1_reg = Z_REG(op1_addr); - val_addr = op2_addr; - } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { - op1_reg = Z_REG(op2_addr); - val_addr = op1_addr; - } else { - | SSE_GET_ZVAL_DVAL result_reg, op1_addr - op1_reg = result_reg; - val_addr = op2_addr; - } - if ((opcode == ZEND_MUL) && - Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { - | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg - } else { - | AVX_MATH opcode, result_reg, op1_reg, val_addr - } + if ((opcode == ZEND_MUL) && + Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { + | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg } else { - zend_jit_addr val_addr; + | AVX_MATH opcode, result_reg, op1_reg, val_addr + } + } else { + zend_jit_addr val_addr; - if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { - | SSE_GET_ZVAL_DVAL result_reg, op2_addr - val_addr = op1_addr; - } else { - | SSE_GET_ZVAL_DVAL result_reg, op1_addr - val_addr = op2_addr; - } - if (same_ops) { - | SSE_MATH_REG opcode, result_reg, result_reg - } else if ((opcode == ZEND_MUL) && - Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { - | SSE_MATH_REG ZEND_ADD, result_reg, result_reg - } else { - | SSE_MATH opcode, result_reg, val_addr - } + if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { + | SSE_GET_ZVAL_DVAL result_reg, op2_addr + val_addr = op1_addr; + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + val_addr = op2_addr; + } + if (same_ops) { + | SSE_MATH_REG opcode, result_reg, result_reg + } else if ((opcode == ZEND_MUL) && + Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { + | SSE_MATH_REG ZEND_ADD, result_reg, result_reg + } else { + | SSE_MATH opcode, result_reg, val_addr } - | SSE_SET_ZVAL_DVAL res_addr, result_reg - |.else - | FPU_GET_ZVAL_DVAL op1_addr - | FPU_MATH opcode, op2_addr - | FPU_SET_ZVAL_DVAL res_addr - |.endif + } + | SSE_SET_ZVAL_DVAL res_addr, result_reg if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { @@ -5800,63 +5630,39 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, i static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) { - |.if SSE - zend_reg tmp_reg = ZREG_XMM0; + zend_reg tmp_reg = ZREG_XMM0; - | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr - | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr - |.else - | FPU_GET_ZVAL_DVAL op2_addr - | FPU_GET_ZVAL_LVAL op1_addr - | fucomip st1 - | fstp st0 - |.endif + | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, 0); } static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) { - zend_bool swap = 0; - - |.if SSE - zend_reg tmp_reg = ZREG_XMM0; + zend_reg tmp_reg = ZREG_XMM0; - | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr - | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr - swap = 1; - |.else - | FPU_GET_ZVAL_LVAL op2_addr - | FPU_GET_ZVAL_DVAL op1_addr - | fucomip st1 - | fstp st0 - |.endif + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr - return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, swap); + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, /* swap */ 1); } static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) { zend_bool swap = 0; - |.if SSE - if (Z_MODE(op1_addr) == IS_REG) { - | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr - } else if (Z_MODE(op2_addr) == IS_REG) { - | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr - swap = 1; - } else { - zend_reg tmp_reg = ZREG_XMM0; + if (Z_MODE(op1_addr) == IS_REG) { + | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr + } else if (Z_MODE(op2_addr) == IS_REG) { + | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr + swap = 1; + } else { + zend_reg tmp_reg = ZREG_XMM0; - | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr - | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr - } - |.else - | FPU_GET_ZVAL_DVAL op2_addr - | FPU_GET_ZVAL_DVAL op1_addr - | fucomip st1 - | fstp st0 - |.endif + | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr + } return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, swap); } @@ -6790,19 +6596,12 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, } if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - |.if SSE - if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { - | vxorps xmm0, xmm0, xmm0 - } else { - | xorps xmm0, xmm0 - } - | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr - |.else - | FPU_GET_ZVAL_DVAL op1_addr - | fldz - | fucomip st1 - | fstp st0 - |.endif + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vxorps xmm0, xmm0, xmm0 + } else { + | xorps xmm0, xmm0 + } + | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr if (set_bool) { if (false_label != (uint32_t)-1) { // JMPZ_EX |