diff options
Diffstat (limited to 'ext/opcache/jit/zend_jit_x86.dasc')
| -rw-r--r-- | ext/opcache/jit/zend_jit_x86.dasc | 10526 |
1 files changed, 10526 insertions, 0 deletions
diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc new file mode 100644 index 0000000000..5023db62da --- /dev/null +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -0,0 +1,10526 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | http://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Dmitry Stogov <dmitry@php.net> | + * | Xinchen Hui <laruence@php.net> | + * +----------------------------------------------------------------------+ + */ + +|.if X64 + |.arch x64 +|.else + |.arch x86 +|.endif + +|.if X64WIN + |.define FP, r14 + |.define IP, r15 + |.define IPl, r15d + |.define RX, r15 // the same as VM IP reused as a general purpos reg + |.define CARG1, rcx // x64/POSIX C call arguments. + |.define CARG2, rdx + |.define CARG3, r8 + |.define CARG4, r9 + |.define CARG1d, ecx + |.define CARG2d, edx + |.define CARG3d, r8d + |.define CARG4d, r9d + |.define FCARG1a, CARG1 // Simulate x86 fastcall. + |.define FCARG2a, CARG2 + |.define FCARG1d, CARG1d + |.define FCARG2d, CARG2d + |.define SPAD, 0x08 // padding for CPU stack alignment + |.define NR_SPAD, 0x58 // padding for CPU stack alignment + |.define SSE, 1 + |.define T3, [r4+0x50] // Used to store old value of IP + |.define T2, [r4+0x48] // Used to store old value of FP + |.define T1, [r4+0x40] + |.define A6, [r4+0x28] // preallocated slot for 6-th argument + |.define A5, [r4+0x20] // preallocated slot for 5-th argument +|.elif X64 + |.define FP, r14 + |.define IP, r15 + |.define IPl, r15d + |.define RX, r15 // the same as VM IP reused as a general purpos reg + |.define CARG1, rdi // x64/POSIX C call arguments. + |.define CARG2, rsi + |.define CARG3, rdx + |.define CARG4, rcx + |.define CARG5, r8 + |.define CARG6, r9 + |.define CARG1d, edi + |.define CARG2d, esi + |.define CARG3d, edx + |.define CARG4d, ecx + |.define CARG5d, r8d + |.define CARG6d, r9d + |.define FCARG1a, CARG1 // Simulate x86 fastcall. + |.define FCARG2a, CARG2 + |.define FCARG1d, CARG1d + |.define FCARG2d, CARG2d + |.define SPAD, 0x08 // padding for CPU stack alignment + |.define NR_SPAD, 0x18 // padding for CPU stack alignment + |.define SSE, 1 + |.define T3, [r4+0x10] // Used to store old value of IP (CALL VM only) + |.define T2, [r4+0x08] // Used to store old value of FP (CALL VM only) + |.define T1, [r4] +|.else + |.define FP, esi + |.define IP, edi + |.define IPl, edi + |.define RX, edi // the same as VM IP reused as a general purpos reg + |.define FCARG1a, ecx // x86 fastcall arguments. + |.define FCARG2a, edx + |.define FCARG1d, ecx + |.define FCARG2d, edx + |.define SPAD, 12 // padding for CPU stack alignment + |.define NR_SPAD, 12 // padding for CPU stack alignment + |.define SSE, 1 + |.define T3, [r4+0x10] // Used to store old value of IP (CALL VM only) + |.define T2, [r4+0x08] // Used to store old value of FP (CALL VM only) + |.define T1, [r4] +|.endif + +|.define HYBRID_SPAD, 16 // padding for stack alignment + +/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to + * guarantee proper alignment of 128-bit SSE data allocated on stack. + * With broken alignment any execution of SSE code, including calls to + * memcpy() and others, may lead to crash. + */ + +#include "Zend/zend_cpuinfo.h" +#include "jit/zend_jit_x86.h" + +/* The generated code may contain tautological comparisons, ignore them. */ +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wtautological-compare" +#endif + +const char* zend_reg_name[] = { +#if defined(__x86_64__) || defined(_M_X64) + "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xm15" +#else + "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" +#endif +}; + +#ifdef HAVE_GCC_GLOBAL_REGS +# define GCC_GLOBAL_REGS 1 +#else +# define GCC_GLOBAL_REGS 0 +#endif + +static uint32_t zend_jit_x86_flags = 0; + +#if ZTS +static size_t tsrm_ls_cache_tcb_offset = 0; +static size_t tsrm_tls_index; +static size_t tsrm_tls_offset; +#endif + +|.type EX, zend_execute_data, FP +|.type OP, zend_op +|.type ZVAL, zval + +|.actionlist dasm_actions + +|.globals zend_lb +static void* dasm_labels[zend_lb_MAX]; + +|.section code, cold_code + +#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0xffffffff) + +#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) + +#define BP_JIT_IS 6 + +|.macro LOAD_ADDR, reg, addr +| .if X64 +|| if (IS_32BIT(addr)) { +| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit> +|| } else { +| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit> +|| } +| .else +| mov reg, ((ptrdiff_t)addr) +| .endif +|.endmacro + +|.macro LOAD_TSRM_CACHE, reg +| .if X64WIN +| gs +| mov reg, aword [0x58] +| mov reg, aword [reg + tsrm_tls_index] +| mov reg, aword [reg + tsrm_tls_offset] +| .elif WIN +| fs +| mov reg, aword [0x2c] +| mov reg, aword [reg + tsrm_tls_index] +| mov reg, aword [reg + tsrm_tls_offset] +| .elif X64 +| fs +|| if (tsrm_ls_cache_tcb_offset) { +| mov reg, aword [tsrm_ls_cache_tcb_offset] +|| } else { +| mov reg, [0x8] +| mov reg, aword [reg + tsrm_tls_index] +| mov reg, aword [reg + tsrm_tls_offset] +|| } +| .else +| gs +|| if (tsrm_ls_cache_tcb_offset) { +| mov reg, aword [tsrm_ls_cache_tcb_offset] +|| } else { +| mov reg, [0x4] +| mov reg, aword [reg + tsrm_tls_index] +| mov reg, aword [reg + tsrm_tls_offset] +|| } +| .endif +|.endmacro + +|.macro LOAD_ADDR_ZTS, reg, struct, field +| .if ZTS +| LOAD_TSRM_CACHE reg +| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] +| .else +| LOAD_ADDR reg, &struct.field +| .endif +|.endmacro + +|.macro SAVE_OPLINE +|| if (GCC_GLOBAL_REGS) { +| mov aword EX->opline, IP +|| } +|.endmacro + +|.macro LOAD_OPLINE +|| if (GCC_GLOBAL_REGS) { +| mov IP, aword EX->opline +|| } +|.endmacro + +|.macro LOAD_IP_ADDR, addr +|| if (GCC_GLOBAL_REGS) { +| LOAD_ADDR IP, addr +|| } else { +| LOAD_ADDR RX, addr +| mov aword EX->opline, RX +|| } +|.endmacro + +|.macro LOAD_IP_ADDR_ZTS, struct, field +| .if ZTS +|| if (GCC_GLOBAL_REGS) { +| LOAD_TSRM_CACHE IP +| mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] +|| } else { +| LOAD_TSRM_CACHE RX +| mov RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] +| mov aword EX->opline, RX +|| } +| .else +| LOAD_IP_ADDR &struct.field +| .endif +|.endmacro + +|.macro GET_IP, reg +|| if (GCC_GLOBAL_REGS) { +| mov reg, IP +|| } else { +| mov reg, aword EX->opline +|| } +|.endmacro + +|.macro ADD_IP, val +|| if (GCC_GLOBAL_REGS) { +| add IP, val +|| } else { +| add aword EX->opline, val +|| } +|.endmacro + +|.macro JMP_IP +|| if (GCC_GLOBAL_REGS) { +| jmp aword [IP] +|| } else { +| mov r0, aword EX:FCARG1a->opline +| jmp aword [r0] +|| } +|.endmacro + +/* In 64-bit build we compare only low 32-bits. + * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full + * comparison would require additinal load of 64-bit address into register. + * This is not a problem at all, while JIT buffer size is less than 4GB. + */ +|.macro CMP_IP, addr +|| if (GCC_GLOBAL_REGS) { +| cmp IPl, addr +|| } else { +| cmp dword EX->opline, addr +|| } +|.endmacro + +|.macro ADDR_OP1, addr_ins, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| addr_ins ((ptrdiff_t)addr) +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| addr_ins tmp_reg +|| } +| .else +| addr_ins ((ptrdiff_t)addr) +| .endif +|.endmacro + +|.macro ADDR_OP2_2, addr_ins, op1, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| addr_ins op1, ((ptrdiff_t)addr) +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| addr_ins op1, tmp_reg +|| } +| .else +| addr_ins op1, ((ptrdiff_t)addr) +| .endif +|.endmacro + +|.macro PUSH_ADDR, addr, tmp_reg +| ADDR_OP1 push, addr, tmp_reg +|.endmacro + +|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE tmp_reg +| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] +| push tmp_reg +| .else +| ADDR_OP1 push, &struct.field, tmp_reg +| .endif +|.endmacro + +|.macro MEM_OP1, mem_ins, prefix, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| mem_ins prefix [addr] +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| mem_ins prefix [tmp_reg] +|| } +| .else +| mem_ins prefix [addr] +| .endif +|.endmacro + +|.macro MEM_OP2_1, mem_ins, prefix, addr, op2, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| mem_ins prefix [addr], op2 +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| mem_ins prefix [tmp_reg], op2 +|| } +| .else +| mem_ins prefix [addr], op2 +| .endif +|.endmacro + +|.macro MEM_OP2_2, mem_ins, op1, prefix, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| mem_ins op1, prefix [addr] +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| mem_ins op1, prefix [tmp_reg] +|| } +| .else +| mem_ins op1, prefix [addr] +| .endif +|.endmacro + +|.macro MEM_OP2_1_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE tmp_reg +| mem_ins prefix [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))], op2 +| .else +| MEM_OP2_1 mem_ins, prefix, &struct.field, op2, tmp_reg +| .endif +|.endmacro + +|.macro MEM_OP2_2_ZTS, mem_ins, op1, prefix, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE tmp_reg +| mem_ins op1, prefix [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] +| .else +| MEM_OP2_2 mem_ins, op1, prefix, &struct.field, tmp_reg +| .endif +|.endmacro + +|.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg +| .if X64 +|| if (IS_32BIT(addr)) { +| mem_ins op1, op2, prefix [addr] +|| } else { +| mov64 tmp_reg, ((ptrdiff_t)addr) +| mem_ins op1, op2, prefix [tmp_reg] +|| } +| .else +| mem_ins op1, op2, prefix [addr] +| .endif +|.endmacro + +|.macro LOAD_BASE_ADDR, reg, base, offset +|| if (offset) { +| lea reg, qword [Ra(base)+offset] +|| } else { +| mov reg, Ra(base) +|| } +|.endmacro + +|.macro PUSH_BASE_ADDR, base, offset, tmp_reg +|| if (offset) { +| lea tmp_reg, qword [Ra(base)+offset] +| push tmp_reg +|| } else { +| push Ra(base) +|| } +|.endmacro + +|.macro EXT_CALL, func, tmp_reg +| .if X64 +|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { +| call qword &func +|| } else { +| LOAD_ADDR tmp_reg, func +| call tmp_reg +|| } +| .else +| call dword &func +| .endif +|.endmacro + +|.macro EXT_JMP, func, tmp_reg +| .if X64 +|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { +| jmp qword &func +|| } else { +| LOAD_ADDR tmp_reg, func +| jmp tmp_reg +|| } +| .else +| jmp dword &func +| .endif +|.endmacro + +|.macro LOAD_ZVAL_ADDR, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| LOAD_ADDR reg, Z_ZV(addr) +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr) +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro PUSH_ZVAL_ADDR, addr, tmp_reg +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| PUSH_ADDR Z_ZV(addr), tmp_reg +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro GET_Z_TYPE_INFO, reg, zv +| mov reg, dword [zv+offsetof(zval,u1.type_info)] +|.endmacro + +|.macro SET_Z_TYPE_INFO, zv, type +| mov dword [zv+offsetof(zval,u1.type_info)], type +|.endmacro + +|.macro GET_ZVAL_TYPE_INFO, reg, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] +|.endmacro + +|.macro SET_ZVAL_TYPE_INFO, addr, type +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type +|.endmacro + +|.macro GET_Z_PTR, reg, zv +| mov reg, aword [zv] +|.endmacro + +|.macro SET_Z_PTR, zv, val +| mov aword [zv], val +|.endmacro + +|.macro GET_ZVAL_PTR, reg, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|.endmacro + +|.macro SET_ZVAL_PTR, addr, val +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val +|.endmacro + +|.macro GET_ZVAL_W2, reg, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] +|.endmacro + +|.macro SET_ZVAL_W2, addr, val +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val +|.endmacro + +|.macro FPU_OP, fp_ins, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| MEM_OP1 fp_ins, qword, Z_ZV(addr), r0 +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| fp_ins qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro FPU_GET_ZVAL_DVAL, addr +| FPU_OP fld, addr +|.endmacro + +|.macro FPU_MATH, opcode, addr +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| FPU_OP fadd, addr +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| FPU_OP fsub, addr +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| FPU_OP fmul, addr +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| FPU_OP fdiv, addr +|| break; +|| } +|.endmacro + +|.macro FPU_SET_ZVAL_DVAL, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| fstp qword [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)] +|.endmacro + +|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| avx_ins op1, op2 +|| } else { +| sse_ins op1, op2 +|| } +|.endmacro + +|.macro SSE_OP, sse_ins, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| MEM_OP2_2 sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), r0 +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro SSE_AVX_OP, sse_ins, avx_ins, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [r0] +|| } +| .else +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro SSE_GET_ZVAL_LVAL, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) +|| } else { +| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) +|| } +|| } else { +|.if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { +| mov64 r0, Z_LVAL_P(Z_ZV(addr)) +|| } else { +| mov r0, Z_LVAL_P(Z_ZV(addr)) +|| } +|.else +| mov r0, Z_LVAL_P(Z_ZV(addr)) +|.endif +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), r0 +|| } else { +| cvtsi2sd, xmm(reg-ZREG_XMM0), r0 +|| } +|| } +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else { +| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } +|| } else if (Z_MODE(addr) == IS_REG) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) +|| } else { +| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) +|| } +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro SSE_GET_ZVAL_DVAL, reg, addr +|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (IS_32BIT(Z_ZV(addr))) { +| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +|| } else { +| LOAD_ADDR r0, Z_ZV(addr) +| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] +|| } +| .else +| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| SSE_AVX_INS movsd, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) +|| } else { +|| ZEND_ASSERT(0); +|| } +|| } +|.endmacro + +|.macro SSE_MATH, opcode, reg, addr +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| SSE_OP addsd, reg, addr +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| SSE_OP subsd, reg, addr +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| SSE_OP mulsd, reg, addr +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| SSE_OP divsd, reg, addr +|| break; +|| } +|.endmacro + +|.macro SSE_MATH_REG, opcode, dst_reg, src_reg +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| } +|.endmacro + +|.macro SSE_SET_ZVAL_DVAL, addr, reg +|| if (Z_MODE(addr) == IS_REG) { +|| if (reg != Z_REG(addr)) { +| SSE_AVX_INS movsd, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) +|| } +|| } else { +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) +|| } +|.endmacro + +|.macro AVX_OP, avx_ins, reg, op1_reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| MEM_OP3_3 avx_ins, xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword, Z_ZV(addr), r0 +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro AVX_MATH, opcode, reg, op1_reg, addr +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| AVX_OP vaddsd, reg, op1_reg, addr +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| AVX_OP vsubsd, reg, op1_reg, addr +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| AVX_OP vmulsd, reg, op1_reg, addr +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| AVX_OP vdivsd, reg, op1_reg, addr +|| break; +|| } +|.endmacro + +|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) +|| break; +|| } +|.endmacro + +|.macro LONG_OP, long_ins, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { +|| if (reg != ZREG_R0) { +| mov64 r0, Z_LVAL_P(Z_ZV(addr)) +| long_ins Ra(reg), r0 +|| } else { +| mov64 r1, Z_LVAL_P(Z_ZV(addr)) +| long_ins Ra(reg), r1 +|| } +|| } else { +| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) +|| } +| .else +| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) +| .endif +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| long_ins Ra(reg), Ra(Z_REG(addr)) +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval +|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { +| .if X64 +|| if (!IS_SIGNED_32BIT(lval)) { +| mov64 r0, lval +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 +|| } else { +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval +|| } +| .else +| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval +| .endif +|| } else if (Z_MODE(op1_addr) == IS_REG) { +| .if X64 +|| if (!IS_SIGNED_32BIT(lval)) { +| mov64 r0, lval +| long_ins Ra(Z_REG(op1_addr)), r0 +|| } else { +| long_ins Ra(Z_REG(op1_addr)), lval +|| } +| .else +| long_ins Ra(Z_REG(op1_addr)), lval +| .endif +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro GET_ZVAL_LVAL, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { +| xor Ra(reg), Ra(reg) +|| } else { +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { +| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) +|| } else { +| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) +|| } +| .else +| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) +| .endif +|| } +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +|| if (reg != Z_REG(addr)) { +| mov Ra(reg), Ra(Z_REG(addr)) +|| } +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro LONG_MATH, opcode, reg, addr +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| LONG_OP add, reg, addr +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| LONG_OP sub, reg, addr +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| LONG_OP imul, reg, addr +|| break; +|| case ZEND_BW_OR: +|| case ZEND_ASSIGN_BW_OR: +| LONG_OP or, reg, addr +|| break; +|| case ZEND_BW_AND: +|| case ZEND_ASSIGN_BW_AND: +| LONG_OP and, reg, addr +|| break; +|| case ZEND_BW_XOR: +|| case ZEND_ASSIGN_BW_XOR: +| LONG_OP xor, reg, addr +|| break; +|| default: +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro LONG_MATH_REG, opcode, dst_reg, src_reg +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| add dst_reg, src_reg +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| sub dst_reg, src_reg +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| imul dst_reg, src_reg +|| break; +|| case ZEND_BW_OR: +|| case ZEND_ASSIGN_BW_OR: +| or dst_reg, src_reg +|| break; +|| case ZEND_BW_AND: +|| case ZEND_ASSIGN_BW_AND: +| and dst_reg, src_reg +|| break; +|| case ZEND_BW_XOR: +|| case ZEND_ASSIGN_BW_XOR: +| xor dst_reg, src_reg +|| break; +|| default: +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro SET_ZVAL_LVAL, addr, lval +|| if (Z_MODE(addr) == IS_REG) { +| mov Ra(Z_REG(addr)), lval +|| } else { +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval +|| } +|.endmacro + +|.macro FPU_LONG_OP, fp_ins, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| MEM_OP1 fp_ins, aword, Z_ZV(addr), r0 +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| fp_ins aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] +|| } else if (Z_MODE(addr) == IS_REG) { +| fp_ins Ra(Z_REG(addr)) +|| } else { +|| ZEND_ASSERT(0); +|| } +|.endmacro + +|.macro FPU_GET_ZVAL_LVAL, addr +| FPU_LONG_OP fild, addr +|.endmacro + +|.macro FPU_MATH_REG, opcode, reg +|| switch (opcode) { +|| case ZEND_ADD: +|| case ZEND_ASSIGN_ADD: +| fadd reg +|| break; +|| case ZEND_SUB: +|| case ZEND_ASSIGN_SUB: +| fsub reg +|| break; +|| case ZEND_MUL: +|| case ZEND_ASSIGN_MUL: +| fmul reg +|| break; +|| case ZEND_DIV: +|| case ZEND_ASSIGN_DIV: +| fdiv reg +|| break; +|| } +|.endmacro + +|.macro ZVAL_COPY_CONST, dst_addr, dst_info, zv, tmp_reg +|| if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; +| .if X64 or SSE +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| } else { +| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| } +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] +| .endif +|| } else { +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] +|| } +| SSE_SET_ZVAL_DVAL dst_addr, dst_reg +| .else +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| FPU_SET_ZVAL_DVAL dst_addr +| .endif +|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { +| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) +|| } else { +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { +|| if (Z_MODE(dst_addr) == IS_REG) { +| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) +|| } else { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL dst_addr, tmp_reg +|| } +|| } else { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +|| } +| .else +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| .endif +|| } +|| } +|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { +|| if (((dst_info & MAY_BE_ANY) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { +| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) +|| } +|| } +|.endmacro + +|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, zv, tmp_reg +|| if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? +|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); +| .if X64 or SSE +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +|| if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { +| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| } else { +| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) +|| } +| .if X64 +|| } else if (!IS_32BIT(zv)) { +| mov64 tmp_reg, ((uintptr_t)zv) +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [tmp_reg] +| .endif +|| } else { +| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] +|| } +| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 +| .else +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| FPU_SET_ZVAL_DVAL dst_addr +|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { +| fldz +|| } else if (Z_DVAL_P(zv) == 1.0) { +| fld1 +|| } else { +| fld qword [zv] +|| } +| FPU_SET_ZVAL_DVAL res_addr +| .endif +|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { +|| if (Z_MODE(dst_addr) == IS_REG) { +| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } else { +| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) +|| } +|| } else { +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { +|| if (Z_MODE(dst_addr) == IS_REG) { +| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) +|| } else { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| SET_ZVAL_LVAL dst_addr, tmp_reg +| SET_ZVAL_LVAL res_addr, tmp_reg +|| } +|| } else if (Z_MODE(dst_addr) == IS_REG) { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) +|| } else { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +|| } +| .else +|| if (Z_MODE(dst_addr) == IS_REG) { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) +|| } else { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) +|| } +| .endif +|| } +|| } +|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { +|| if (((dst_info & MAY_BE_ANY) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { +| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) +|| } +|| } +|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { +| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv) +|| } +|.endmacro + +/* the same as above, but "src" may overlap with "tmp_reg1" */ +|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 +|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { +|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { +|| if (Z_MODE(src_addr) == IS_REG) { +|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) +|| } +|| } else if (Z_MODE(dst_addr) == IS_REG) { +| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr +|| } else { +| GET_ZVAL_LVAL tmp_reg2, src_addr +| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) +|| } +|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { +| .if X64 or SSE +|| if (Z_MODE(src_addr) == IS_REG) { +| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) +|| } else if (Z_MODE(dst_addr) == IS_REG) { +| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr +|| } else { +| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr +| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +|| } +| .else +| FPU_GET_ZVAL_DVAL src_addr +| FPU_SET_ZVAL_DVAL dst_addr +| .endif +|| } else if (!(src_info & MAY_BE_DOUBLE)) { +| GET_ZVAL_PTR Ra(tmp_reg2), src_addr +| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) +|| } else { +| .if X64 +| GET_ZVAL_PTR Ra(tmp_reg2), src_addr +| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) +| .else +|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) { +| GET_ZVAL_W2 Ra(tmp_reg2), src_addr +| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) +| GET_ZVAL_PTR Ra(tmp_reg2), src_addr +| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) +|| } else { +| GET_ZVAL_PTR Ra(tmp_reg2), src_addr +| GET_ZVAL_W2 Ra(tmp_reg1), src_addr +| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) +| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) +|| } +| .endif +|| } +|| } +|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && +|| has_concrete_type(src_info & MAY_BE_ANY)) { +|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { +|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { +|| zend_uchar type = concrete_type(src_info); +| SET_ZVAL_TYPE_INFO dst_addr, type +|| } +|| } +|| } else { +| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr +| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) +|| } +|.endmacro + +|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2 +|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { +|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { +|| if (Z_MODE(src_addr) == IS_REG) { +|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) +|| } +|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr)) +|| } +|| } else if (Z_MODE(dst_addr) == IS_REG) { +| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr +|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { +| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) +|| } +|| } else if (Z_MODE(res_addr) == IS_REG) { +| GET_ZVAL_LVAL Z_REG(res_addr), src_addr +| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) +|| } else { +| GET_ZVAL_LVAL tmp_reg2, src_addr +| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) +| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2) +|| } +|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { +| .if X64 or SSE +|| if (Z_MODE(src_addr) == IS_REG) { +| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) +| SSE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) +|| } else if (Z_MODE(dst_addr) == IS_REG) { +| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr +| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| SSE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr +| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) +|| } else { +| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr +| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 +| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 +|| } +| .else +| FPU_GET_ZVAL_DVAL src_addr +| FPU_STROE dst_addr +| FPU_GET_ZVAL_DVAL src_addr +| FPU_SET_ZVAL_DVAL res_addr +| .endif +|| } else if (!(src_info & MAY_BE_DOUBLE)) { +| GET_ZVAL_PTR Ra(tmp_reg2), src_addr +| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) +| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) +|| } else { +| .if X64 +| GET_ZVAL_PTR Ra(tmp_reg2), src_addr +| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) +| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) +| .else +|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) { +| GET_ZVAL_W2 Ra(tmp_reg2), src_addr +| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) +| SET_ZVAL_W2 res_addr, Ra(tmp_reg2) +| GET_ZVAL_PTR Ra(tmp_reg2), src_addr +| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) +| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) +|| } else { +| GET_ZVAL_PTR Ra(tmp_reg2), src_addr +| GET_ZVAL_W2 Ra(tmp_reg1), src_addr +| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) +| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) +| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) +| SET_ZVAL_W2 res_addr, Ra(tmp_reg1) +|| } +| .endif +|| } +|| } +|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && +|| has_concrete_type(src_info & MAY_BE_ANY)) { +|| zend_uchar type = concrete_type(src_info); +|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { +|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { +| SET_ZVAL_TYPE_INFO dst_addr, type +|| } +|| } +|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { +| SET_ZVAL_TYPE_INFO res_addr, type +|| } +|| } else { +| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr +| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) +| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1) +|| } +|.endmacro + +|.macro IF_TYPE, type, val, label +| cmp type, val +| je label +|.endmacro + +|.macro IF_NOT_TYPE, type, val, label +| cmp type, val +| jne label +|.endmacro + +|.macro IF_Z_TYPE, zv, val, label +| IF_TYPE byte [zv + offsetof(zval, u1.v.type)], val, label +|.endmacro + +|.macro IF_NOT_Z_TYPE, zv, val, label +| IF_NOT_TYPE byte [zv + offsetof(zval, u1.v.type)], val, label +|.endmacro + +|.macro CMP_ZVAL_TYPE, addr, val +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val +|.endmacro + +|.macro IF_ZVAL_TYPE, addr, val, label +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label +|.endmacro + +|.macro IF_NOT_ZVAL_TYPE, addr, val, label +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label +|.endmacro + +|.macro IF_FLAGS, type_flags, mask, label +| test type_flags, mask +| jnz label +|.endmacro + +|.macro IF_NOT_FLAGS, type_flags, mask, label +| test type_flags, mask +| jz label +|.endmacro + +|.macro IF_REFCOUNTED, type_flags, label +| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label +|.endmacro + +|.macro IF_NOT_REFCOUNTED, type_flags, label +| IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label +|.endmacro + +|.macro IF_ZVAL_FLAGS, addr, mask, label +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label +|.endmacro + +|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label +|.endmacro + +|.macro IF_ZVAL_REFCOUNTED, addr, label +| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label +|.endmacro + +|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label +| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label +|.endmacro + +|.macro GC_ADDREF, zv +| add dword [zv], 1 +|.endmacro + +|.macro GC_DELREF, zv +| sub dword [zv], 1 +|.endmacro + +|.macro IF_GC_MAY_NOT_LEAK, ptr, tmp_reg, label +| mov tmp_reg, dword [ptr + 4] +| and tmp_reg, (GC_INFO_MASK | (GC_COLLECTABLE << GC_FLAGS_SHIFT)) +| cmp tmp_reg, (GC_COLLECTABLE << GC_FLAGS_SHIFT) +| jne label +|.endmacro + +|.macro ADDREF_CONST, zv, tmp_reg +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| add dword [tmp_reg], 1 +|| } else { +| add dword [Z_LVAL_P(zv)], 1 +|| } +| .else +| add dword [Z_LVAL_P(zv)], 1 +| .endif +|.endmacro + +|.macro ADDREF_CONST_2, zv, tmp_reg +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| add dword [tmp_reg], 2 +|| } else { +| add dword [Z_LVAL_P(zv)], 2 +|| } +| .else +| add dword [Z_LVAL_P(zv)], 2 +| .endif +|.endmacro + +|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| IF_NOT_REFCOUNTED type_flags_reg, >1 +|| } +| GC_ADDREF value_ptr_reg +|1: +|| } +|.endmacro + +|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| IF_NOT_REFCOUNTED type_flags_reg, >1 +|| } +| add dword [value_ptr_reg], 2 +|1: +|| } +|.endmacro + +|.macro ZVAL_DEREF, reg, info +|| if (info & MAY_BE_REF) { +| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 +| GET_Z_PTR reg, reg +| add reg, offsetof(zend_reference, val) +|1: +|| } +|.endmacro + +|.macro SAVE_VALID_OPLINE, op +|| if (op == last_valid_opline) { +| SAVE_OPLINE +|| } else { +| ADDR_OP2_2 mov, aword EX->opline, op, r0 +|| } +|.endmacro + +// zval should be in FCARG1a +|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a +|| do { +|| if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); +|| if (type == IS_STRING && !ZEND_DEBUG) { +| EXT_CALL _efree, r0 +|| break; +|| } else if (type == IS_ARRAY) { +|| if (opline) { +| SAVE_VALID_OPLINE opline +|| } +| EXT_CALL zend_array_destroy, r0 +|| break; +|| } else if (type == IS_OBJECT) { +|| if (opline) { +| SAVE_VALID_OPLINE opline +|| } +| EXT_CALL zend_objects_store_del, r0 +|| break; +|| } +|| } +|| if (opline) { +| SAVE_VALID_OPLINE opline +|| } +| EXT_CALL rc_dtor_func, r0 +|| } while(0); +|.endmacro + +|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, safe, opline +|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { +|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| // if (Z_REFCOUNTED_P(cv)) { +|| if (cold) { +| IF_ZVAL_REFCOUNTED addr, >1 +|.cold_code +|1: +|| } else { +| IF_NOT_ZVAL_REFCOUNTED addr, >4 +|| } +|| } +| // if (!Z_DELREF_P(cv)) { +| GET_ZVAL_PTR FCARG1a, addr +| GC_DELREF FCARG1a +|| if (RC_MAY_BE_1(op_info)) { +|| if (RC_MAY_BE_N(op_info)) { +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +| jnz >3 +|| } else { +| jnz >4 +|| } +|| } +|| if (safe) { +| // ZVAL_NULL(cv); +| SET_ZVAL_TYPE_INFO addr, IS_NULL +|| } +| // zval_dtor_func(r); +| ZVAL_DTOR_FUNC op_info, opline +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +| jmp >4 +|| } +|3: +|| } +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +|| if ((op_info) & MAY_BE_REF) { +|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val)); +| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 +| IF_NOT_ZVAL_REFCOUNTED ref_addr, >4 +| GET_ZVAL_PTR FCARG1a, ref_addr +|1: +|| } +| IF_GC_MAY_NOT_LEAK FCARG1a, eax, >4 +| // gc_possible_root(Z_COUNTED_P(z)) +| EXT_CALL gc_possible_root, r0 +|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { +| jmp >4 +|.code +|| } +|| } else if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { +| jmp >4 +|.code +|| } +|4: +|| } +|.endmacro + +|.macro FREE_OP, op_type, op, op_info, cold, op_array, opline +|| if (op_type & (IS_VAR|IS_TMP_VAR)) { +| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, 0, opline +|| } +|.endmacro + +|.macro SEPARATE_ZVAL_NOREF, addr, op_info, cold +|| if ((op_info & MAY_BE_ARRAY) && RC_MAY_BE_N(op_info)) { +|| if (cold) { +| IF_ZVAL_TYPE addr, IS_ARRAY, >1 +|.cold_code +|1: +|| } else { +| IF_NOT_ZVAL_TYPE addr, IS_ARRAY, >2 +|| } +| GET_ZVAL_PTR r0, addr +|| if (RC_MAY_BE_1(op_info)) { +| cmp dword [r0], 1 // if (GC_REFCOUNTED() > 1) +| jbe >2 +|| } +| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, >1 +| GC_DELREF r0 +|1: +|| if (Z_REG(addr) == ZREG_FCARG1a) { +| mov aword T1, FCARG1a // save +|| } else { +| LOAD_ZVAL_ADDR FCARG1a, addr +|| } +| EXT_CALL zval_copy_ctor_func, r0 +|| if (Z_REG(addr) == ZREG_FCARG1a) { +| mov FCARG1a, aword T1 // restore +|| } +|| if (cold) { +| jmp >2 +|.code +|| } +|2: +|| } +|.endmacro + +|.macro SEPARATE_ARRAY, addr, op_info, cold +|| if (RC_MAY_BE_N(op_info)) { +|| zend_reg tmp_reg; +|| +|| tmp_reg = (Z_REG(addr) == ZREG_FCARG1a) ? ZREG_R0 : ZREG_FCARG1a; +| GET_ZVAL_LVAL tmp_reg, addr +|| if (RC_MAY_BE_1(op_info)) { +| cmp dword [Ra(tmp_reg)], 1 // if (GC_REFCOUNTED() > 1) +|| if (cold) { +| ja >1 +|.cold_code +|1: +|| } else { +| jbe >2 +|| } +|| } +| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, >1 +| GC_DELREF Ra(tmp_reg) +|1: +|| if (Z_REG(addr) == ZREG_FCARG1a) { +| mov aword T1, FCARG1a // save +|| } else { +| LOAD_ZVAL_ADDR FCARG1a, addr +|| } +| EXT_CALL zval_copy_ctor_func, r0 +|| if (Z_REG(addr) == ZREG_FCARG1a) { +| mov FCARG1a, aword T1 // restore +|| } +|| if (RC_MAY_BE_1(op_info)) { +|| if (cold) { +| jmp >2 +|.code +|| } +|| } +|2: +|| } +| GET_ZVAL_LVAL ZREG_FCARG1a, addr +|.endmacro + +|.macro EFREE_REG_24, op_array, opline +||#if ZEND_DEBUG +|| const char *filename = op_array->filename ? op_array->filename->val : NULL; +| LOAD_ADDR FCARG2a, filename +| .if X64WIN +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| mov aword A5, 0 +| EXT_CALL _efree, r0 +| .elif X64 +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| xor CARG5, CARG5 +| EXT_CALL _efree, r0 +| .else +| sub r4, 4 +| push 0 +| push 0 +| push opline->lineno +| EXT_CALL _efree, r0 +| add r4, 4 +| .endif +||#else +||#ifdef HAVE_BUILTIN_CONSTANT_P +| EXT_CALL _efree_24, r0 +||#else +| EXT_CALL _efree, r0 +||#endif +||#endif +|.endmacro + +|.macro EFREE_24, ptr, op_array, opline +| mov FCARG1a, ptr +| EFREE_REG_24 op_array, opline +|.endmacro + +|.macro EMALLOC, size, op_array, opline +||#if ZEND_DEBUG +|| const char *filename = op_array->filename ? op_array->filename->val : NULL; +| mov FCARG1a, size +| LOAD_ADDR FCARG2a, filename +| .if X64WIN +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| mov aword A5, 0 +| EXT_CALL _emalloc, r0 +| .elif X64 +| mov CARG3d, opline->lineno +| xor CARG4, CARG4 +| xor CARG5, CARG5 +| EXT_CALL _emalloc, r0 +| .else +| sub r4, 4 +| push 0 +| push 0 +| push opline->lineno +| EXT_CALL _emalloc, r0 +| add r4, 4 +| .endif +||#else +||#ifdef HAVE_BUILTIN_CONSTANT_P +|| if (size == 24) { +| EXT_CALL _emalloc_24, r0 +|| } else { +| mov FCARG1a, size +| EXT_CALL _emalloc, r0 +|| } +||#else +| mov FCARG1a, size +| EXT_CALL _emalloc, r0 +||#endif +||#endif +|.endmacro + +|.macro OBJ_RELEASE, reg, tmp_reg, exit_label +| GC_DELREF reg +| jne >1 +| // zend_objects_store_del(obj); +| mov FCARG1a, reg +| EXT_CALL zend_objects_store_del, r0 +| jmp exit_label +|1: +| IF_GC_MAY_NOT_LEAK reg, tmp_reg, >1 +| // gc_possible_root(obj) +| mov FCARG1a, reg +| EXT_CALL gc_possible_root, r0 +|1: +|.endmacro + +|.macro UNDEFINED_OFFSET, opline +|| if (opline == last_valid_opline) { +| call ->undefined_offset_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->undefined_offset +|| } +|.endmacro + +|.macro UNDEFINED_INDEX, opline +|| if (opline == last_valid_opline) { +| call ->undefined_index_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->undefined_index +|| } +|.endmacro + +|.macro CANNOT_ADD_ELEMENT, opline +|| if (opline == last_valid_opline) { +| call ->cannot_add_element_ex +|| } else { +| SAVE_VALID_OPLINE, opline +| call ->cannot_add_element +|| } +|.endmacro + +static zend_bool reuse_ip; +static zend_bool delayed_call_chain; +static uint32_t delayed_call_level; +static const zend_op *last_valid_opline; +static int jit_return_label; + +/* bit helpers */ + +/* from http://aggregate.org/MAGIC/ */ +static uint32_t ones32(uint32_t x) +{ + x -= ((x >> 1) & 0x55555555); + x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); + x = (((x >> 4) + x) & 0x0f0f0f0f); + x += (x >> 8); + x += (x >> 16); + return x & 0x0000003f; +} + +static uint32_t floor_log2(uint32_t x) +{ + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + return ones32(x) - 1; +} + +static zend_bool is_power_of_two(uint32_t x) +{ + return !(x & (x - 1)); +} + +static zend_bool has_concrete_type(uint32_t value_type) +{ + if (value_type & MAY_BE_UNDEF) { + value_type |= MAY_BE_NULL; + } + value_type &= MAY_BE_ANY; + return is_power_of_two (value_type); +} + +static uint32_t concrete_type(uint32_t value_type) +{ + return floor_log2(value_type & MAY_BE_ANY); +} + +static inline zend_bool is_signed(double d) +{ + return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; +} + +static int zend_jit_interrupt_handler_stub(dasm_State **Dst) +{ + |->interrupt_handler: + | //EG(vm_interrupt) = 0; + | MEM_OP2_1_ZTS mov, byte, executor_globals, vm_interrupt, 0, r0 + | //if (EG(timed_out)) { + | MEM_OP2_1_ZTS cmp, byte, executor_globals, timed_out, 0, r0 + | je >1 + | //zend_timeout(0); + | xor FCARG1d, FCARG1d + | EXT_CALL zend_timeout, r0 + |1: + | //} else if (zend_interrupt_function) { + if (zend_interrupt_function) { + | SAVE_OPLINE + | //zend_interrupt_function(execute_data); + |.if X64 + | mov CARG1, FP + | EXT_CALL zend_interrupt_function, r0 + |.else + | sub r4, 12 + | push FP + | EXT_CALL zend_interrupt_function, r0 + | add r4, 16 + |.endif + | //ZEND_VM_ENTER(); + | //execute_data = EG(current_execute_data); + | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 + | LOAD_OPLINE + } + | //ZEND_VM_CONTINUE() + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + | JMP_IP + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + | JMP_IP + } else { + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 1 // ZEND_VM_ENTER + | ret + } + + return 1; +} + +static int zend_jit_exception_handler_stub(dasm_State **Dst) +{ + |->exception_handler: + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + const void *handler = zend_get_opcode_handler_func(EG(exception_op)); + + | add r4, HYBRID_SPAD + | EXT_CALL handler, r0 + | JMP_IP + } else { + const void *handler = EG(exception_op)->handler; + + if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + } else { + | mov FCARG1a, FP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + } + | EXT_JMP handler, r0 + } + + return 1; +} + +static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) +{ + |->exception_handler_undef: + | SET_Z_TYPE_INFO FP + r0, IS_UNDEF + | jmp ->exception_handler + + return 1; +} + +static int zend_jit_leave_function_stub(dasm_State **Dst) +{ + |->leave_function_handler: + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | test FCARG1d, ZEND_CALL_TOP + | jnz >1 + | EXT_CALL zend_jit_leave_nested_func_helper, r0 + | add r4, HYBRID_SPAD // stack alignment + | JMP_IP + |1: + | EXT_CALL zend_jit_leave_top_func_helper, r0 + | add r4, HYBRID_SPAD // stack alignment + | JMP_IP + } else { + if (GCC_GLOBAL_REGS) { + | add r4, SPAD + } else { + | mov FCARG2a, FP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD + } + | test FCARG1d, ZEND_CALL_TOP + | jnz >1 + | EXT_JMP zend_jit_leave_nested_func_helper, r0 + |1: + | EXT_JMP zend_jit_leave_top_func_helper, r0 + } + + return 1; +} + +static int zend_jit_leave_throw_stub(dasm_State **Dst) +{ + |->leave_throw_handler: + | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { + if (GCC_GLOBAL_REGS) { + | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION + | je >5 + | // EG(opline_before_exception) = opline; + | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0 + |5: + | // opline = EG(exception_op); + | LOAD_IP_ADDR_ZTS executor_globals, exception_op + | // HANDLE_EXCEPTION() + | jmp ->exception_handler + } else { + | GET_IP FCARG1a + | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION + | je >5 + | // EG(opline_before_exception) = opline; + | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, FCARG1a, r0 + |5: + | // opline = EG(exception_op); + | LOAD_IP_ADDR_ZTS executor_globals, exception_op + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 2 // ZEND_VM_LEAVE + | ret + } + + return 1; +} + +static int zend_jit_icall_throw_stub(dasm_State **Dst) +{ + |->icall_throw_handler: + | // zend_throw_exception_internal(NULL); + |.if X64 + | xor CARG1, CARG1 + | EXT_CALL zend_throw_exception_internal, r0 + |.else + | sub r4, 12 + | push 0 + | EXT_CALL zend_throw_exception_internal, r0 + | add r4, 16 + |.endif + | // HANDLE_EXCEPTION() + | jmp ->exception_handler + + return 1; +} + +static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) +{ + |->throw_cannot_pass_by_ref: + | mov r0, EX->opline + |.if X64 + | movsxd r1, dword OP:r0->result.var + |.else + | mov r1, OP:r0->result.var + |.endif + | SET_Z_TYPE_INFO RX+r1, IS_UNDEF + | mov RX, r0 + |.if X64 + | xor CARG1, CARG1 + | LOAD_ADDR CARG2, "Cannot pass parameter %d by reference" + | mov CARG3d, dword OP:r0->op2.num + | EXT_CALL zend_throw_error, r0 + |.else + | mov r1, dword OP:r0->op2.num + | sub r4, 4 + | push r1 + | push "Cannot pass parameter %d by reference" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | cmp byte OP:RX->op1_type, IS_TMP_VAR + | jne >9 + |.if X64 + | movsxd r0, dword OP:RX->op1.var + |.else + | mov r0, OP:RX->op1.var + |.endif + | add r0, FP + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, 0, NULL + |9: + | jmp ->exception_handler + + return 1; +} + +static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) +{ + |->undefined_offset_ex: + | SAVE_OPLINE + | jmp ->undefined_offset + + return 1; +} + +static int zend_jit_undefined_offset_stub(dasm_State **Dst) +{ + |->undefined_offset: + |.if X64WIN + | sub r4, 0x28 + |.elif X64 + | sub r4, 8 + |.else + | sub r4, 12 + |.endif + | mov r0, EX->opline + |.if X64 + | movsxd r1, dword OP:r0->result.var + |.else + | mov r1, OP:r0->result.var + |.endif + | cmp byte OP:r0->op2_type, IS_CONST + | SET_Z_TYPE_INFO FP + r1, IS_NULL + | jne >2 + |.if X64 + | movsxd r1, dword OP:r0->op2.constant + | add r0, r1 + |.else + | mov r0, aword OP:r0->op2.zv + |.endif + | jmp >3 + |2: + |.if X64 + | movsxd r0, dword OP:r0->op2.var + |.else + | mov r0, OP:r0->op2.var + |.endif + | add r0, FP + |3: + |.if X64WIN + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | mov CARG3, aword [r0] + | EXT_CALL zend_error, r0 + | add r4, 0x28 // stack alignment + |.elif X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined offset: " ZEND_LONG_FMT + | mov CARG3, aword [r0] + | EXT_CALL zend_error, r0 + | add r4, 8 // stack alignment + |.else + | sub r4, 4 + | push aword [r0] + | push "Undefined offset: " ZEND_LONG_FMT + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 28 + |.endif + | ret + + return 1; +} + +static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) +{ + |->undefined_index_ex: + | SAVE_OPLINE + | jmp ->undefined_index + + return 1; +} + +static int zend_jit_undefined_index_stub(dasm_State **Dst) +{ + |->undefined_index: + |.if X64WIN + | sub r4, 0x28 + |.elif X64 + | sub r4, 8 + |.else + | sub r4, 12 + |.endif + | mov r0, EX->opline + |.if X64 + | movsxd r1, dword OP:r0->result.var + |.else + | mov r1, OP:r0->result.var + |.endif + | cmp byte OP:r0->op2_type, IS_CONST + | SET_Z_TYPE_INFO FP + r1, IS_NULL + | jne >2 + |.if X64 + | movsxd r1, dword OP:r0->op2.constant + | add r0, r1 + |.else + | mov r0, aword OP:r0->op2.zv + |.endif + | jmp >3 + |2: + |.if X64 + | movsxd r0, dword OP:r0->op2.var + |.else + | mov r0, OP:r0->op2.var + |.endif + | add r0, FP + |3: + |.if X64WIN + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | mov CARG3, aword [r0] + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + | add r4, 0x28 + |.elif X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Undefined index: %s" + | mov CARG3, aword [r0] + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_error, r0 + | add r4, 8 + |.else + | sub r4, 4 + | mov r0, aword [r0] + | add r0, offsetof(zend_string, val) + | push r0 + | push "Undefined index: %s" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 28 + |.endif + | ret + + return 1; +} + +static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) +{ + |->cannot_add_element_ex: + | SAVE_OPLINE + | jmp ->cannot_add_element + + return 1; +} + +static int zend_jit_cannot_add_element_stub(dasm_State **Dst) +{ + |->cannot_add_element: + |.if X64WIN + | sub r4, 0x28 + |.elif X64 + | sub r4, 8 + |.else + | sub r4, 12 + |.endif + | mov r0, EX->opline + | cmp byte OP:r0->result_type, IS_UNUSED + | jz >1 + |.if X64 + | movsxd r0, dword OP:r0->result.var + |.else + | mov r0, OP:r0->result.var + |.endif + | SET_Z_TYPE_INFO FP + r0, IS_NULL + |1: + |.if X64WIN + | mov CARG1, E_WARNING + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_CALL zend_error, r0 + | add r4, 0x28 + |.elif X64 + | mov CARG1, E_WARNING + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_CALL zend_error, r0 + | add r4, 8 + |.else + | sub r4, 8 + | push "Cannot add element to the array as the next element is already occupied" + | push E_WARNING + | EXT_CALL zend_error, r0 + | add r4, 28 + |.endif + | ret + + return 1; +} + +static int zend_jit_not_obj_stub(dasm_State **Dst) +{ + |->not_obj: + |.if X64 + | xor CARG1, CARG1 + | LOAD_ADDR CARG2, "Using $this when not in object context" + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 8 + | push "Using $this when not in object context" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | jmp ->exception_handler + return 1; +} + +static int zend_jit_undefined_function_stub(dasm_State **Dst) +{ + |->undefined_function: + | mov r0, aword EX->opline + |.if X64 + | xor CARG1, CARG1 + | LOAD_ADDR CARG2, "Call to undefined function %s()" + | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)] + | mov CARG3, aword [r0 + CARG3] + | add CARG3, offsetof(zend_string, val) + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 4 + | mov r0, aword [r0 + offsetof(zend_op, op2.zv)] + | mov r0, aword [r0] + | add r0, offsetof(zend_string, val) + | push r0 + | push "Call to undefined function %s()" + | push 0 + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | jmp ->exception_handler + return 1; +} + +static int zend_jit_negative_shift_stub(dasm_State **Dst) +{ + |->negative_shift: + |.if X64 + |.if WIN + | LOAD_ADDR CARG1, &zend_ce_arithmetic_error + | mov CARG1, aword [CARG1] + |.else + | LOAD_ADDR CARG1, zend_ce_arithmetic_error + |.endif + | LOAD_ADDR CARG2, "Bit shift by negative number" + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 8 + | push "Bit shift by negative number" + |.if WIN + | LOAD_ADDR r0, &zend_ce_arithmetic_error + | push aword [r0] + |.else + | PUSH_ADDR zend_ce_arithmetic_error, r0 + |.endif + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | jmp ->exception_handler + return 1; +} + +static int zend_jit_mod_by_zero_stub(dasm_State **Dst) +{ + |->mod_by_zero: + |.if X64 + |.if WIN + | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error + | mov CARG1, aword [CARG1] + |.else + | LOAD_ADDR CARG1, zend_ce_division_by_zero_error + |.endif + | LOAD_ADDR CARG2, "Modulo by zero" + | EXT_CALL zend_throw_error, r0 + |.else + | sub r4, 8 + | push "Modulo by zero" + |.if WIN + | LOAD_ADDR r0, &zend_ce_division_by_zero_error + | push aword [r0] + |.else + | PUSH_ADDR zend_ce_division_by_zero_error, r0 + |.endif + | EXT_CALL zend_throw_error, r0 + | add r4, 16 + |.endif + | jmp ->exception_handler + return 1; +} + +static int zend_jit_double_one_stub(dasm_State **Dst) +{ + |->one: + |.dword 0, 0x3ff00000 + return 1; +} + +static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) +{ + |->hybrid_runtime_jit: + | EXT_CALL zend_runtime_jit, r0 + | JMP_IP + return 1; +} + +static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) +{ + |->hybrid_profile_jit: + | // ++zend_jit_profile_counter; + | .if X64 + | LOAD_ADDR r0, &zend_jit_profile_counter + | inc aword [r0] + | .else + | inc aword [&zend_jit_profile_counter] + | .endif + | // op_array = (zend_op_array*)EX(func); + | mov r0, EX->func + | // ++ZEND_COUNTER_INFO(op_array) + | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] +#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR + | mov r2, aword [r2] +#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET + | xor r1, r1 + | test r2, 1 + | jz >1 + | MEM_OP2_2_ZTS mov, r1, aword, compiler_globals, map_ptr_base, r1 + | sub r1, 1 + |1: + | mov r2, aword [r1 + r2] +#else +# error "Unknown ZEND_MAP_PTR_KIND" +#endif + | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] + | // handler = (const void*)ZEND_FUNC_INFO(op_array); + | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | // return ((zend_vm_opcode_handler_t)handler)(); + | jmp r0 + return 1; +} + +/* + * This code is based Mike Pall's "Hashed profile counters" idea, implemented + * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual + * property disclosure and research opportunities" email + * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html + * + * In addition we use a variation of Knuth's multiplicative hash function + * described at https://code.i-harness.com/en/q/a21ce + * + * uint64_t hash(uint64_t x) { + * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; + * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; + * x = x ^ (x >> 31); + * return x; + * } + * + * uint_32_t hash(uint32_t x) { + * x = ((x >> 16) ^ x) * 0x45d9f3b; + * x = ((x >> 16) ^ x) * 0x45d9f3b; + * x = (x >> 16) ^ x; + * return x; + * } + * + */ +static int zend_jit_hybrid_func_counter_stub(dasm_State **Dst) +{ + |->hybrid_func_counter: + | mov r0, EX->func + | mov r1, aword [r0 + offsetof(zend_op_array, function_name)] + | test r1, r1 + | jne >1 + | mov r0, aword [r0 + offsetof(zend_op_array, filename)] + |1: + | shr r0, 3 + | mov r1, r0 + | .if X64 + | shr r0, 30 + | xor r0, r1 + | mov64 r1, 0xbf58476d1ce4e5b9 + | imul r0, r1 + | mov r1, r0 + | shr r0, 27 + | xor r0, r1 + | mov64 r1, 0x94d049bb133111eb + | imul r0, r1 + | mov r1, r0 + | shr r0, 31 + | xor r1, r0 + | and r1, ZEND_HOT_COUNTERS_COUNT-1 + | LOAD_ADDR r0, &zend_jit_hot_counters + | sub word [r0+r1*2], ZEND_JIT_HOT_FUNC_COST + | .else + | shr r0, 16 + | xor r0, r1 + | imul r0, 0x45d9f3b + | mov r1, r0 + | shr r0, 16 + | xor r0, r1 + | imul r0, 0x45d9f3b + | mov r1, r0 + | shr r0, 16 + | xor r1, r0 + | and r1, ZEND_HOT_COUNTERS_COUNT-1 + | sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_FUNC_COST + | .endif + | jle >1 + | mov r1, EX->func + | GET_IP r0 + | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] + | // divide by sizeof(zend_op) + | .if X64 + || ZEND_ASSERT(sizeof(zend_op) == 32); + | sar r0, 5 + | .else + || ZEND_ASSERT(sizeof(zend_op) == 28); + | sar r0, 2 + | imul r0, 0xb6db6db7 + | .endif + | mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | .if X64 + | jmp aword [r1+r0*8] + | .else + | jmp aword [r1+r0*4] + | .endif + |1: + | .if X64 + | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT + | .else + | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT + | .endif + | mov FCARG1a, FP + | GET_IP FCARG2a + | EXT_CALL zend_jit_hot_func, r0 + | JMP_IP + return 1; +} + +static int zend_jit_hybrid_loop_counter_stub(dasm_State **Dst) +{ + |->hybrid_loop_counter: + | mov r0, EX->func + | mov r1, aword [r0 + offsetof(zend_op_array, function_name)] + | test r1, r1 + | jne >1 + | mov r0, aword [r0 + offsetof(zend_op_array, filename)] + |1: + | shr r0, 3 + | mov r1, r0 + | .if X64 + | shr r0, 30 + | xor r0, r1 + | mov64 r1, 0xbf58476d1ce4e5b9 + | imul r0, r1 + | mov r1, r0 + | shr r0, 27 + | xor r0, r1 + | mov64 r1, 0x94d049bb133111eb + | imul r0, r1 + | mov r1, r0 + | shr r0, 31 + | xor r1, r0 + | and r1, ZEND_HOT_COUNTERS_COUNT-1 + | LOAD_ADDR r0, &zend_jit_hot_counters + | sub word [r0+r1*2], ZEND_JIT_HOT_LOOP_COST + | .else + | shr r0, 16 + | xor r0, r1 + | imul r0, 0x45d9f3b + | mov r1, r0 + | shr r0, 16 + | xor r0, r1 + | imul r0, 0x45d9f3b + | mov r1, r0 + | shr r0, 16 + | xor r1, r0 + | and r1, ZEND_HOT_COUNTERS_COUNT-1 + | sub word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_LOOP_COST + | .endif + | jle >1 + | mov r1, EX->func + | GET_IP r0 + | sub r0, aword [r1 + offsetof(zend_op_array, opcodes)] + | // divide by sizeof(zend_op) + | .if X64 + || ZEND_ASSERT(sizeof(zend_op) == 32); + | sar r0, 5 + | .else + || ZEND_ASSERT(sizeof(zend_op) == 28); + | sar r0, 2 + | imul r0, 0xb6db6db7 + | .endif + | mov r1, aword [r1 + offsetof(zend_op_array, reserved[zend_func_info_rid])] + | .if X64 + | jmp aword [r1+r0*8] + | .else + | jmp aword [r1+r0*4] + | .endif + |1: + | .if X64 + | mov word [r0+r1*2], ZEND_JIT_HOT_COUNTER_INIT + | .else + | mov word [r1*2+&zend_jit_hot_counters], ZEND_JIT_HOT_COUNTER_INIT + | .endif + | mov FCARG1a, FP + | GET_IP FCARG2a + | EXT_CALL zend_jit_hot_func, r0 + | JMP_IP + return 1; +} + +#ifdef CONTEXT_THREADED_JIT +static int zend_jit_context_threaded_call_stub(dasm_State **Dst) +{ + |->context_threaded_call: + | pop r0 + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + | jmp aword [IP] + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + | jmp aword [IP] + } else { + ZEND_ASSERT(0); + // TODO: context threading can't work without GLOBAL REGS because we have to change + // the value of execute_data in execute_ex() + | mov FCARG1a, FP + | mov r0, aword [FP] + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | jmp aword [r0] + } + return 1; +} +#endif + +static const zend_jit_stub zend_jit_stubs[] = { + JIT_STUB(interrupt_handler), + JIT_STUB(exception_handler), + JIT_STUB(exception_handler_undef), + JIT_STUB(leave_function), + JIT_STUB(leave_throw), + JIT_STUB(icall_throw), + JIT_STUB(throw_cannot_pass_by_ref), + JIT_STUB(undefined_offset), + JIT_STUB(undefined_index), + JIT_STUB(cannot_add_element), + JIT_STUB(undefined_offset_ex), + JIT_STUB(undefined_index_ex), + JIT_STUB(cannot_add_element_ex), + JIT_STUB(not_obj), + JIT_STUB(undefined_function), + JIT_STUB(negative_shift), + JIT_STUB(mod_by_zero), + JIT_STUB(double_one), +#ifdef CONTEXT_THREADED_JIT + JIT_STUB(context_threaded_call), +#endif +}; + +#if ZTS && defined(ZEND_WIN32) +extern uint32_t _tls_index; +extern char *_tls_start; +extern char *_tls_end; +#endif + +static int zend_jit_setup(void) +{ + |.if SSE + if (!zend_cpu_supports(ZEND_CPU_FEATURE_SSE2)) { + zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); + return FAILURE; + } + if (zend_jit_cpu_flags & ZEND_JIT_CPU_AVX) { + if (zend_cpu_supports(ZEND_CPU_FEATURE_AVX)) { + zend_jit_x86_flags |= ZEND_JIT_CPU_AVX; + } + } + |.endif + +#if ZTS +# ifdef _WIN64 + tsrm_tls_index = _tls_index * sizeof(void*); + + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ + do { + void ***tls_mem = ((void***)__readgsqword(0x58))[_tls_index]; + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + tsrm_tls_offset = offset; + break; + } + tls_mem++; + offset += sizeof(void*); + } + if (offset >= size) { + // TODO: error message ??? + return FAILURE; + } + } while(0); +# elif ZEND_WIN32 + tsrm_tls_index = _tls_index * sizeof(void*); + + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ + do { + void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + tsrm_tls_offset = offset; + break; + } + tls_mem++; + offset += sizeof(void*); + } + if (offset >= size) { + // TODO: error message ??? + return FAILURE; + } + } while(0); +# elif defined(__GNUC__) && defined(__x86_64__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { + size_t *ti; + + __asm__( + "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" + : "=a" (ti)); + tsrm_tls_offset = ti[1]; + tsrm_tls_index = ti[0] * 16; + } +# elif defined(__GNUC__) && defined(__i386__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { +#if 1 + size_t ret; + + asm ("leal _tsrm_ls_cache@ntpoff,%0\n" + : "=a" (ret)); + tsrm_ls_cache_tcb_offset = ret; +#else + size_t *ti, _ebx, _ecx, _edx; + + __asm__( + "call 1f\n" + ".subsection 1\n" + "1:\tmovl (%%esp), %%ebx\n\t" + "ret\n" + ".previous\n\t" + "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" + "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" + "call ___tls_get_addr@plt\n\t" + "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" + : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); + tsrm_tls_offset = ti[1]; + tsrm_tls_index = ti[0] * 8; +#endif + } +# endif +#endif + + return SUCCESS; +} + +static int zend_jit_align_func(dasm_State **Dst) +{ + reuse_ip = 0; + delayed_call_chain = 0; + last_valid_opline = NULL; + jit_return_label = -1; + |.align 16 + return 1; +} + +static int zend_jit_prologue(dasm_State **Dst) +{ + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | sub r4, HYBRID_SPAD + } else if (GCC_GLOBAL_REGS) { + | sub r4, SPAD // stack alignment + } else { + | sub r4, NR_SPAD // stack alignment + | mov aword T2, FP // save FP + | mov aword T3, RX // save IP + | mov FP, FCARG1a + } + return 1; +} + +static int zend_jit_label(dasm_State **Dst, unsigned int label) +{ + |=>label: + return 1; +} + +static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) +{ + | // call->prev_execute_data = EX(call); + if (call_level == 1) { + | mov aword EX:RX->prev_execute_data, 0 + } else { + | mov r0, EX->call + | mov EX:RX->prev_execute_data, r0 + } + | // EX(call) = call; + | mov EX->call, RX + + delayed_call_chain = 0; + + return 1; +} + +static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) +{ + if (!last_valid_opline) { + | LOAD_IP_ADDR opline + } else if (last_valid_opline != opline) { + if (GCC_GLOBAL_REGS) { + | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); + } else { + | LOAD_IP_ADDR opline + } + } + return 1; +} + +static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) +{ + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + if (!zend_jit_set_ip(Dst, opline)) { + return 0; + } + last_valid_opline = opline; + reuse_ip = 0; + return 1; +} + +static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline) +{ + if (zend_interrupt_function) { +#if 0 + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 + | jne ->interrupt_handler +#else + | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 + if (last_valid_opline == opline) { + | jne ->interrupt_handler + } else { + | jne >1 + |.cold_code + |1: + | LOAD_IP_ADDR opline + | jmp ->interrupt_handler + |.code + } + return 1; +#endif + } + return 1; +} + +static int zend_jit_check_exception(dasm_State **Dst) +{ + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | jne ->exception_handler + return 1; +} + +static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) +{ + if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | mov r0, opline->result.var + | jne ->exception_handler_undef + return 1; + } + return zend_jit_check_exception(Dst); +} + +static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) +{ + const void *handler; + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + handler = zend_get_opcode_handler_func(opline); + } else { + handler = opline->handler; + } + + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, FP + } + | EXT_CALL handler, r0 + if (may_throw) { + zend_jit_check_exception(Dst); + } + last_valid_opline++; + + /* Skip the following OP_DATA */ + switch (opline->opcode) { + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_OBJ: + case ZEND_ASSIGN_STATIC_PROP: + case ZEND_ASSIGN_STATIC_PROP_REF: + case ZEND_ASSIGN_OBJ_REF: + last_valid_opline++; + break; + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + case ZEND_ASSIGN_MOD: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_CONCAT: + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_POW: + if (opline->extended_value) { + last_valid_opline++; + } + break; + default: + break; + } + + return 1; +} + +static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) +{ + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + if (opline->opcode == ZEND_DO_UCALL || + opline->opcode == ZEND_DO_FCALL_BY_NAME || + opline->opcode == ZEND_DO_FCALL || + opline->opcode == ZEND_RETURN) { + + /* Use inlined HYBRID VM handler */ + const void *handler = opline->handler; + + | add r4, HYBRID_SPAD + | EXT_JMP handler, r0 + } else { + const void *handler = zend_get_opcode_handler_func(opline); + + | EXT_CALL handler, r0 + | add r4, HYBRID_SPAD + | JMP_IP + } + } else { + const void *handler = opline->handler; + + if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + } else { + | mov FCARG1a, FP + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + } + | EXT_JMP handler, r0 + } + last_valid_opline = NULL; + return 1; +} + +static int zend_jit_set_opline(dasm_State **Dst, const zend_op *target_opline) +{ + if (!reuse_ip) { + last_valid_opline = target_opline; + } + return 1; +} + +static int zend_jit_reset_opline(dasm_State **Dst, const zend_op *target_opline) +{ + last_valid_opline = NULL; + return 1; +} + +static void zend_jit_start_reuse_ip(void) { + last_valid_opline = NULL; + reuse_ip = 1; +} + +static void zend_jit_stop_reuse_ip(void) { + reuse_ip = 0; +} + +static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) +{ + | jmp =>target_label + return 1; +} + +static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) +{ + | CMP_IP next_opline + | jne =>target_label + + last_valid_opline = next_opline; + + return 1; +} + +#ifdef CONTEXT_THREADED_JIT +static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) +{ + if (!zend_jit_handler(Dst, opline, 1)) return 0; + if (opline->opcode == ZEND_DO_UCALL) { + | call ->context_threaded_call + } else { + const zend_op *next_opline = opline + 1; + + | CMP_IP next_opline + | je =>next_block + | call ->context_threaded_call + } + return 1; +} +#endif + +static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) +{ +#ifdef CONTEXT_THREADED_JIT + return zend_jit_context_threaded_call(Dst, opline, next_block); +#else + return zend_jit_tail_handler(Dst, opline); +#endif +} + +static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_bool set_type) +{ + ZEND_ASSERT(Z_MODE(src) == IS_REG); + ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); + + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) + if (set_type) { + | SET_ZVAL_TYPE_INFO dst, IS_LONG + } + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | .if SSE + | SSE_SET_ZVAL_DVAL dst, Z_REG(src) + | .else + || ZEND_ASSERT(0); + | .endif + if (set_type) { + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE + } + } else { + ZEND_ASSERT(0); + } + return 1; +} + +static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +{ + ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); + ZEND_ASSERT(Z_MODE(dst) == IS_REG); + + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | GET_ZVAL_LVAL Z_REG(dst), src + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | .if SSE + | SSE_GET_ZVAL_DVAL Z_REG(dst), src + | .else + || ZEND_ASSERT(0); + | .endif + } else { + ZEND_ASSERT(0); + } + return 1; +} + +static int zend_jit_store_ssa_var(dasm_State **Dst, zend_ssa *ssa, int var, zend_reg reg) +{ + zend_jit_addr src = ZEND_ADDR_REG(reg); + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var)); + uint32_t info = ssa->var_info[var].type; + + return zend_jit_spill_store(Dst, src, dst, info, 1); +} + +static int zend_jit_store_ssa_var_if_necessary(dasm_State **Dst, zend_ssa *ssa, zend_lifetime_interval **ra, zend_jit_addr src, int var, int old_var) +{ + if (Z_MODE(src) == IS_REG && ra[var] && ra[var]->store) { + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var)); + uint32_t info = ssa->var_info[var].type; + zend_bool set_type = 1; + + if (old_var >= 0) { + uint32_t old_info = ssa->var_info[old_var].type; + if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == + (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { + if (!ra[old_var] || ra[old_var]->load || ra[old_var]->store) { + set_type = 0; + } + } + } + return zend_jit_spill_store(Dst, src, dst, info, set_type); + } + return 1; +} + +static int zend_jit_load_ssa_var(dasm_State **Dst, zend_ssa *ssa, int var, zend_reg reg) +{ + zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, ssa->vars[var].var)); + zend_jit_addr dst = ZEND_ADDR_REG(reg); + uint32_t info = ssa->var_info[var].type; + + return zend_jit_load_reg(Dst, src, dst, info); +} + +static int zend_jit_update_regs(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_lifetime_interval *src_ival) +{ + if (src != dst) { + if (Z_MODE(src) == IS_REG) { + if (Z_MODE(dst) == IS_REG) { + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | .if SSE + | SSE_AVX_INS movsd, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) + | .else + || ZEND_ASSERT(0); + | .endif + } else { + ZEND_ASSERT(0); + } + } else if (Z_MODE(dst) == IS_MEM_ZVAL) { + if (!src_ival->load && !src_ival->store) { + if (!zend_jit_spill_store(Dst, src, dst, info, 1)) { + return 0; + } + } + } else { + ZEND_ASSERT(0); + } + } else if (Z_MODE(src) == IS_MEM_ZVAL) { + if (Z_MODE(dst) == IS_REG) { + if (!zend_jit_load_reg(Dst, src, dst, info)) { + return 0; + } + } else { + ZEND_ASSERT(0); + } + } else { + ZEND_ASSERT(0); + } + } + return 1; +} + +static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info, op1_def_info, res_use_info = 0; + zend_jit_addr op1_addr, op1_def_addr, res_addr = 0; + + op1_info = OP1_INFO(); + if (opline->op1_type != IS_CV || !(op1_info & MAY_BE_LONG)) { + goto fallback; + } + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1); + if (opline->result_type != IS_UNUSED) { + res_use_info = RES_USE_INFO(); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); + } + + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 + } + if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + } + if (op1_addr != op1_def_addr) { + if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op1_def_addr) == IS_REG && + (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { + if (!zend_jit_update_regs(Dst, res_addr, op1_def_addr, MAY_BE_LONG, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { + return 0; + } + } else { + if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, MAY_BE_LONG, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { + return 0; + } + } + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | LONG_OP_WITH_CONST add, op1_def_addr, Z_L(1) + } else { + | LONG_OP_WITH_CONST sub, op1_def_addr, Z_L(1) + } + op1_def_info = OP1_DEF_INFO(); + + if ((op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { + | jo >1 + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + } + |.cold_code + |1: + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + |.if X64 + | mov64 rax, 0x43e0000000000000 + | SET_ZVAL_LVAL op1_def_addr, rax + |.else + | SET_ZVAL_LVAL op1_def_addr, 0 + | SET_ZVAL_W2 op1_def_addr, 0x41e00000 + |.endif + } else { + |.if X64 + | mov64 rax, 0xc3e0000000000000 + | SET_ZVAL_LVAL op1_def_addr, rax + |.else + | SET_ZVAL_LVAL op1_def_addr, 0x00200000 + | SET_ZVAL_W2 op1_def_addr, 0xc1e00000 + |.endif + } + | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 + } + | jmp >3 + |.code + } else { + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 + } + } + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + |.cold_code + |2: + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | SAVE_VALID_OPLINE opline + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL + op1_info |= MAY_BE_NULL; + } + |2: + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + + | // ZVAL_DEREF(var_ptr); + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 + | GET_Z_PTR FCARG2a, FCARG1a + | cmp aword [FCARG2a + offsetof(zend_reference, sources.ptr)], 0 + | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] + | jz >2 + |.if X64 + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR CARG3, res_addr + } else { + | mov CARG3, 0 + } + |.else + | sub r4, 12 + if (RETURN_VALUE_USED(opline)) { + | PUSH_ZVAL_ADDR res_addr, r0 + } else { + | push 0 + } + |.endif + if (opline->opcode == ZEND_PRE_INC) { + | EXT_CALL zend_jit_pre_inc_typed_ref, r0 + } else if (opline->opcode == ZEND_PRE_DEC) { + | EXT_CALL zend_jit_pre_dec_typed_ref, r0 + } else if (opline->opcode == ZEND_POST_INC) { + | EXT_CALL zend_jit_post_inc_typed_ref, r0 + } else if (opline->opcode == ZEND_POST_DEC) { + | EXT_CALL zend_jit_post_dec_typed_ref, r0 + } else { + ZEND_ASSERT(0); + } + |.if not(X64) + | add r4, 12 + |.endif + zend_jit_check_exception(Dst); + | jmp >3 + |2: + } + + if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { + if (opline->result_type != IS_UNUSED) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + + | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 + | TRY_ADDREF op1_info, ah, r2 + } + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | EXT_CALL increment_function, r0 + } else { + | EXT_CALL decrement_function, r0 + } + } else { + zend_reg tmp_reg; + + if ((opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC)) { + if (opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 + } + } + |.if SSE + if (Z_MODE(op1_def_addr) == IS_REG) { + tmp_reg = Z_REG(op1_def_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + tmp_reg = Z_REG(op1_addr); + } else { + tmp_reg = ZREG_XMM0; + } + | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] + } else { + | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] + } + } else { + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] + } else { + | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] + } + } + | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg + |.else + | FPU_GET_ZVAL_DVAL op1_addr + | fld1 + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | fadd + } else { + | fsub + } + | FPU_SET_ZVAL_DVAL op1_def_addr + |.endif + } + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 + | TRY_ADDREF op1_def_info, ah, r1 + } + | jmp >3 + |.code + } + |3: + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, op1_def_addr, ssa->ops[opline - op_array->opcodes].op1_def, ssa->ops[opline - op_array->opcodes].op1_use)) { + return 0; + } + if (opline->result_type != IS_UNUSED) { + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { + return 0; + } + } + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_math_long_long(dasm_State **Dst, + zend_op_array *op_array, + zend_ssa *ssa, + const zend_op *opline, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info) +{ + zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; + + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_R0; + } + + if (opline->opcode == ZEND_MUL && + ((Z_MODE(op2_addr) == IS_CONST_ZVAL && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) || + (Z_MODE(op1_addr) == IS_CONST_ZVAL && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | GET_ZVAL_LVAL result_reg, op1_addr + | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + } else { + | GET_ZVAL_LVAL result_reg, op2_addr + | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + } + } else if (opline->opcode == ZEND_DIV && + (Z_MODE(op2_addr) == IS_CONST_ZVAL && + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { + | GET_ZVAL_LVAL result_reg, op1_addr + | shr Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + } else { + | GET_ZVAL_LVAL result_reg, op1_addr + if (same_ops && opline->opcode != ZEND_DIV) { + | LONG_MATH_REG opline->opcode, Ra(result_reg), Ra(result_reg) + } else { + | LONG_MATH opline->opcode, result_reg, op2_addr + } + } + if ((res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, op_array, ssa)) { + | jo >1 + |.cold_code + |1: + |.if SSE + zend_reg tmp_reg1 = ZREG_XMM0; + zend_reg tmp_reg2 = ZREG_XMM1; + + | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH_REG opline->opcode, tmp_reg1, tmp_reg1, tmp_reg2 + } else { + | SSE_MATH_REG opline->opcode, tmp_reg1, tmp_reg2 + } + | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 + |.else + | FPU_GET_ZVAL_LVAL op2_addr + | FPU_GET_ZVAL_LVAL op1_addr + | FPU_MATH_REG opline->opcode, st1 + | FPU_SET_ZVAL_DVAL res_addr + |.endif + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } + | jmp >2 + |.code + | SET_ZVAL_LVAL res_addr, Ra(result_reg) + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG + } + } + |2: + } else if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + | SET_ZVAL_LVAL res_addr, Ra(result_reg) + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG + } + } + } + + return 1; +} + +static int zend_jit_math_long_double(dasm_State **Dst, + zend_op_array *op_array, + zend_ssa *ssa, + const zend_op *opline, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + |.if SSE + zend_reg result_reg = + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; + + | SSE_GET_ZVAL_LVAL result_reg, op1_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH opline->opcode, result_reg, result_reg, op2_addr + } else { + | SSE_MATH opline->opcode, result_reg, op2_addr + } + | SSE_SET_ZVAL_DVAL res_addr, result_reg + |.else + | FPU_GET_ZVAL_LVAL op1_addr + | FPU_MATH opline->opcode, op2_addr + | FPU_SET_ZVAL_DVAL res_addr + |.endif + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } + } + + return 1; +} + +static int zend_jit_math_double_long(dasm_State **Dst, + zend_op_array *op_array, + zend_ssa *ssa, + const zend_op *opline, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + |.if SSE + zend_reg result_reg; + + if (zend_is_commutative(opline->opcode)) { + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else { + result_reg = ZREG_XMM0; + } + | SSE_GET_ZVAL_LVAL result_reg, op2_addr + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | AVX_MATH opline->opcode, result_reg, result_reg, op1_addr + } else { + | SSE_MATH opline->opcode, result_reg, op1_addr + } + } else { + zend_reg tmp_reg; + + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + tmp_reg = ZREG_XMM0; + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + tmp_reg = ZREG_XMM0; + } else { + result_reg = ZREG_XMM0; + tmp_reg = ZREG_XMM1; + } + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + zend_reg op1_reg; + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + op1_reg = result_reg; + } + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | AVX_MATH_REG opline->opcode, result_reg, op1_reg, tmp_reg + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | SSE_MATH_REG opline->opcode, result_reg, tmp_reg + } + } + | SSE_SET_ZVAL_DVAL res_addr, result_reg + |.else + | FPU_GET_ZVAL_LVAL op2_addr + if (zend_is_commutative(opline->opcode)) { + | FPU_MATH opline->opcode, op1_addr + } else { + | FPU_GET_ZVAL_DVAL op1_addr + | FPU_MATH_REG opline->opcode, st1 + } + | FPU_SET_ZVAL_DVAL res_addr + |.endif + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } + } + } + + return 1; +} + +static int zend_jit_math_double_double(dasm_State **Dst, + zend_op_array *op_array, + zend_ssa *ssa, + const zend_op *opline, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + + |.if SSE + zend_reg result_reg; + + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_XMM0; + } + + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + zend_reg op1_reg; + zend_jit_addr val_addr; + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + val_addr = op2_addr; + } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opline->opcode)) { + op1_reg = Z_REG(op2_addr); + val_addr = op1_addr; + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + op1_reg = result_reg; + val_addr = op2_addr; + } + if ((opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) && + Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { + | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg + } else { + | AVX_MATH opline->opcode, result_reg, op1_reg, val_addr + } + } else { + zend_jit_addr val_addr; + + if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opline->opcode)) { + | SSE_GET_ZVAL_DVAL result_reg, op2_addr + val_addr = op1_addr; + } else { + | SSE_GET_ZVAL_DVAL result_reg, op1_addr + val_addr = op2_addr; + } + if (same_ops) { + | SSE_MATH_REG opline->opcode, result_reg, result_reg + } else if ((opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) && + Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { + | SSE_MATH_REG ZEND_ADD, result_reg, result_reg + } else { + | SSE_MATH opline->opcode, result_reg, val_addr + } + } + | SSE_SET_ZVAL_DVAL res_addr, result_reg + |.else + | FPU_GET_ZVAL_DVAL op1_addr + | FPU_MATH opline->opcode, op2_addr + | FPU_SET_ZVAL_DVAL res_addr + |.endif + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE + } + } + } + + return 1; +} + +static int zend_jit_math_helper(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + uint32_t res_var, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info) +/* Labels: 1,2,3,4,5,6 */ +{ + zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 + } + } + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { + if (op2_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 + |.cold_code + |1: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 + } + if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | jmp >5 + |.code + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 + } + } + if (!zend_jit_math_long_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_info, res_use_info)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + |3: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops) { + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 + } else { + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 + } + } + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | jmp >5 + } + if (!same_ops) { + |1: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 + } + if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | jmp >5 + } + |.code + } + } else if ((op1_info & MAY_BE_DOUBLE) && + !(op1_info & MAY_BE_LONG) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op2_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 + } + } + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + } + if (!same_ops && (op2_info & MAY_BE_LONG)) { + if (op2_info & MAY_BE_DOUBLE) { + |.cold_code + } + |1: + if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 + } + if (!zend_jit_math_double_long(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + if (op2_info & MAY_BE_DOUBLE) { + | jmp >5 + |.code + } + } + } else if ((op2_info & MAY_BE_DOUBLE) && + !(op2_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 + } + if (op1_info & MAY_BE_DOUBLE) { + if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op1_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 + } + } + if (!zend_jit_math_double_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + } + if (!same_ops && (op1_info & MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + } + |1: + if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 + } + if (!zend_jit_math_long_double(Dst, op_array, ssa, opline, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + | jmp >5 + |.code + } + } + } + + |5: + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + |.cold_code + } + |6: + | SAVE_VALID_OPLINE opline + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + | LOAD_ZVAL_ADDR FCARG1a, real_addr + } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr + } + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif + if (opline->opcode == ZEND_ADD || opline->opcode == ZEND_ASSIGN_ADD) { + | EXT_CALL add_function, r0 + } else if (opline->opcode == ZEND_SUB || opline->opcode == ZEND_ASSIGN_SUB) { + | EXT_CALL sub_function, r0 + } else if (opline->opcode == ZEND_MUL || opline->opcode == ZEND_ASSIGN_MUL) { + | EXT_CALL mul_function, r0 + } else if (opline->opcode == ZEND_DIV || opline->opcode == ZEND_ASSIGN_DIV) { + | EXT_CALL div_function, r0 + } else { + ZEND_ASSERT(0); + } + |.if not(X64) + | add r4, 12 + |.endif + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { + return 0; + } + } + if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + | jmp <5 + |.code + } + } + + return 1; +} + +static int zend_jit_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info, op2_info, res_use_info; + zend_jit_addr op1_addr, op2_addr, res_addr; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + res_use_info = RES_USE_INFO(); + + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + goto fallback; + } + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); + + if (opline->result_type == IS_TMP_VAR && + (opline+1)->opcode == ZEND_SEND_VAL && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + /* Eliminate the following SEND_VAL */ + (*opnum)++; + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); + res_use_info = -1; + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); + } + + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, RES_INFO(), res_use_info)) { + return 0; + } + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { + return 0; + } + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_long_math_helper(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + int op1_ssa_var, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + int op2_ssa_var, + uint32_t res_var, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info) +/* Labels: 6 */ +{ + zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; + zend_uchar opcode = opline->opcode; + zval tmp; + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 + } + if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 + } + + if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL && + op1_ssa_var >= 0 && + ssa->var_info[op1_ssa_var].has_range && + ssa->var_info[op1_ssa_var].range.min >= 0) { + zend_long l = Z_LVAL_P(Z_ZV(op2_addr)); + + if (zend_long_is_power_of_two(l)) { + /* Optimisation for mod of power of 2 */ + opcode = ZEND_BW_AND; + ZVAL_LONG(&tmp, l - 1); + op2_addr = ZEND_ADDR_CONST_ZVAL(&tmp); + } + } + + if (opcode == ZEND_MOD) { + result_reg = ZREG_RAX; + } else if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && zend_ssa_is_last_use(op_array, ssa, ssa->ops[opline-op_array->opcodes].op1_use, opline-op_array->opcodes)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_R0; + } + + if (opcode == ZEND_SL || opcode == ZEND_ASSIGN_SL) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { + if (EXPECTED(op2_lval > 0)) { + | xor Ra(result_reg), Ra(result_reg) + } else { + | SAVE_VALID_OPLINE opline + | jmp ->negative_shift + } + } else { + | GET_ZVAL_LVAL result_reg, op1_addr + | shl Ra(result_reg), op2_lval + } + } else { + if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { + | GET_ZVAL_LVAL ZREG_RCX, op2_addr + } + if (op2_ssa_var < 0 || + !ssa->var_info[op2_ssa_var].has_range || + ssa->var_info[op2_ssa_var].range.min < 0 || + ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) { + | cmp r1, (SIZEOF_ZEND_LONG*8) + | jae >1 + |.cold_code + |1: + | cmp r1, 0 + | mov Ra(result_reg), 0 + | jg >1 + | SAVE_VALID_OPLINE opline + | jmp ->negative_shift + |.code + } + | GET_ZVAL_LVAL result_reg, op1_addr + | shl Ra(result_reg), cl + |1: + } + } else if (opcode == ZEND_SR || opcode == ZEND_ASSIGN_SR) { + | GET_ZVAL_LVAL result_reg, op1_addr + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { + if (EXPECTED(op2_lval > 0)) { + | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 + } else { + | SAVE_VALID_OPLINE opline + | jmp ->negative_shift + } + } else { + | sar Ra(result_reg), op2_lval + } + } else { + if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { + | GET_ZVAL_LVAL ZREG_RCX, op2_addr + } + if (op2_ssa_var < 0 || + !ssa->var_info[op2_ssa_var].has_range || + ssa->var_info[op2_ssa_var].range.min < 0 || + ssa->var_info[op2_ssa_var].range.max >= SIZEOF_ZEND_LONG * 8) { + | cmp r1, (SIZEOF_ZEND_LONG*8) + | jae >1 + |.cold_code + |1: + | cmp r1, 0 + | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 + | jg >1 + | SAVE_VALID_OPLINE opline + | jmp ->negative_shift + |.code + } + |1: + | sar Ra(result_reg), cl + } + } else if (opcode == ZEND_MOD || opcode == ZEND_ASSIGN_MOD) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (op2_lval == 0) { + | SAVE_VALID_OPLINE opline + | jmp ->mod_by_zero + } else if (op2_lval == -1) { + | xor Ra(result_reg), Ra(result_reg) + } else { + result_reg = ZREG_RDX; + | GET_ZVAL_LVAL ZREG_RAX, op1_addr + | GET_ZVAL_LVAL ZREG_RCX, op2_addr + |.if X64 + | cqo + |.else + | cdq + |.endif + | idiv Ra(ZREG_RCX) + } + } else { + if (op2_ssa_var < 0 || + !ssa->var_info[op2_ssa_var].has_range || + (ssa->var_info[op2_ssa_var].range.min <= 0 && + ssa->var_info[op2_ssa_var].range.max >= 0)) { + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 + } else if (Z_MODE(op2_addr) == IS_REG) { + | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) + } + | jz >1 + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | jmp ->mod_by_zero + |.code + } + + //TODO: add check for -1 ??? + + result_reg = ZREG_RDX; + | GET_ZVAL_LVAL ZREG_RAX, op1_addr + |.if X64 + | cqo + |.else + | cdq + |.endif + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] + } else if (Z_MODE(op2_addr) == IS_REG) { + | idiv Ra(Z_REG(op2_addr)) + } + } + } else if (same_ops) { + | GET_ZVAL_LVAL result_reg, op1_addr + | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) + } else { + | GET_ZVAL_LVAL result_reg, op1_addr + | LONG_MATH opcode, result_reg, op2_addr + } + + | SET_ZVAL_LVAL res_addr, Ra(result_reg) + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG + } + } + } + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG)) { + |5: + |.cold_code + } + |6: + | SAVE_VALID_OPLINE opline + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + | LOAD_ZVAL_ADDR FCARG1a, real_addr + } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr + } + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif + if (opline->opcode == ZEND_BW_OR || opline->opcode == ZEND_ASSIGN_BW_OR) { + | EXT_CALL bitwise_or_function, r0 + } else if (opline->opcode == ZEND_BW_AND || opline->opcode == ZEND_ASSIGN_BW_AND) { + | EXT_CALL bitwise_and_function, r0 + } else if (opline->opcode == ZEND_BW_XOR || opline->opcode == ZEND_ASSIGN_BW_XOR) { + | EXT_CALL bitwise_xor_function, r0 + } else if (opline->opcode == ZEND_SL || opline->opcode == ZEND_ASSIGN_SL) { + | EXT_CALL shift_left_function, r0 + } else if (opline->opcode == ZEND_SR || opline->opcode == ZEND_ASSIGN_SR) { + | EXT_CALL shift_right_function, r0 + } else if (opline->opcode == ZEND_MOD || opline->opcode == ZEND_ASSIGN_MOD) { + | EXT_CALL mod_function, r0 + } else { + ZEND_ASSERT(0); + } + |.if not(X64) + | add r4, 12 + |.endif + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { + return 0; + } + } + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG)) { + | jmp <5 + |.code + } + } + + return 1; +} + +static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info, op2_info, res_use_info; + zend_jit_addr op1_addr, op2_addr, res_addr; + int op1_ssa_var, op2_ssa_var; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_ssa_var = ssa->ops[opline - op_array->opcodes].op1_use; + op2_ssa_var = ssa->ops[opline - op_array->opcodes].op2_use; + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + res_use_info = RES_USE_INFO(); + + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & MAY_BE_LONG) || + !(op2_info & MAY_BE_LONG)) { + goto fallback; + } + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); + + if (opline->result_type == IS_TMP_VAR && + (opline+1)->opcode == ZEND_SEND_VAL && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + /* Eliminate the following SEND_VAL */ + (*opnum)++; + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); + res_use_info = -1; + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ssa->ops[opline - op_array->opcodes].result_def); + } + + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->result.var, res_addr, RES_INFO(), res_use_info)) { + return 0; + } + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { + return 0; + } + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_concat_helper(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + zend_jit_addr res_addr, + uint32_t res_info) +{ +#if 1 + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 + } + if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 + } + if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { + if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr + } + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + | EXT_CALL zend_jit_fast_assign_concat_helper, r0 + } else { + if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr + } + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif + | EXT_CALL zend_jit_fast_concat_helper, r0 + |.if not(X64) + | add r4, 12 + |.endif + } + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline + |5: + } + if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || + (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + |.cold_code + |6: + } +#endif + | SAVE_VALID_OPLINE opline + if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1a, res_addr + } + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif + | EXT_CALL concat_function, r0 + |.if not(X64) + | add r4, 12 + |.endif + | FREE_OP op1_type, op1, op1_info, 0, op_array, opline + | FREE_OP op2_type, op2, op2_info, 0, op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } +#if 1 + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + | jmp <5 + |.code + } + } +#endif + + return 1; +} + +static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, res_addr; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + if (!(op1_info & MAY_BE_STRING) || + !(op2_info & MAY_BE_STRING)) { + goto fallback; + } + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + + if (opline->result_type == IS_TMP_VAR && + (opline+1)->opcode == ZEND_SEND_VAL && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + /* Eliminate the following SEND_VAL */ + (*opnum)++; + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + } + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, RES_INFO()); + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint32_t found, uint32_t not_found) +/* Labels: 1,2,3,4,5 */ +{ + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + if (op2_info & MAY_BE_LONG) { + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 + } + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long val = Z_LVAL_P(Z_ZV(op2_addr)); + if (val >= 0 && val < HT_MAX_SIZE) { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED + | jz >4 // HASH_FIND + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) + |.if X64 + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + if (val == 0) { + | test r0, r0 + } else { + | cmp r0, val + } + |.else + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val + |.endif + if (type == BP_JIT_IS) { + | jbe >9 // NOT_FOUND + } else { + | jbe >2 // NOT_FOUND + } + | // _ret = &_ht->arData[_h].val; + | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] + if (val != 0) { + | add r0, val * sizeof(Bucket) + } + if (type == BP_JIT_IS) { + | jmp >5 + } else { + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + } + } + } else { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED + | jz >4 // HASH_FIND + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) + |.if X64 + | movsxd r0, dword [FCARG1a + offsetof(zend_array, nNumUsed)] + | cmp r0, FCARG2a + |.else + | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a + |.endif + if (type == BP_JIT_IS) { + | jbe >9 // NOT_FOUND + } else { + | jbe >2 // NOT_FOUND + } + | // _ret = &_ht->arData[_h].val; + |.if X64 + | mov r0, FCARG2a + | shl r0, 5 + |.else + | imul r0, FCARG2a, sizeof(Bucket) + |.endif + | add r0, aword [FCARG1a + offsetof(zend_array, arData)] + if (type == BP_JIT_IS) { + | jmp >5 + } else { + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + } + } + } + switch (type) { + case BP_JIT_IS: + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + |4: + } + | EXT_CALL zend_hash_index_find, r0 + | test r0, r0 + | jz >9 // NOT_FOUND + if (op2_info & MAY_BE_STRING) { + | jmp >5 + } + break; + case BP_VAR_R: + case BP_VAR_IS: + case BP_VAR_UNSET: + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long val = Z_LVAL_P(Z_ZV(op2_addr)); + if (val >= 0 && val < HT_MAX_SIZE) { + | jmp >2 // NOT_FOUND + } + } else { + | jmp >2 // NOT_FOUND + } + |4: + } + | EXT_CALL zend_hash_index_find, r0 + | test r0, r0 + | jz >2 // NOT_FOUND + |.cold_code + |2: + switch (type) { + case BP_VAR_R: + | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + | // retval = &EG(uninitialized_zval); + | UNDEFINED_OFFSET opline + | jmp >9 + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + | // retval = &EG(uninitialized_zval); + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL + | jmp >9 + break; + default: + ZEND_ASSERT(0); + } + |.code + break; + case BP_VAR_RW: + |2: + | SAVE_VALID_OPLINE opline + | // zend_error(E_NOTICE,"Undefined offset: " ZEND_LONG_FMT, hval); + | //retval = zend_hash_index_update(ht, hval, &EG(uninitialized_zval)); + | EXT_CALL zend_jit_fetch_dimension_rw_long_helper, r0 + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + | jmp >8 + |4: + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_jit_hash_index_lookup_rw, r0 + } + break; + case BP_VAR_W: + |2: + | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); + |.if X64 + | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval + |.else + | sub r4, 12 + | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 + |.endif + | EXT_CALL zend_hash_index_add_new, r0 + |.if not(X64) + | add r4, 12 + |.endif + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + | jmp >8 + |4: + | EXT_CALL zend_jit_hash_index_lookup_w, r0 + } + break; + default: + ZEND_ASSERT(0); + } + + if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { + | jmp >8 + } + } + + if (op2_info & MAY_BE_STRING) { + |3: + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 + } + | // offset_key = Z_STR_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr + | // retval = zend_hash_find(ht, offset_key); + switch (type) { + case BP_JIT_IS: + if (opline->op2_type != IS_CONST) { + | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' + | jle >1 + |.cold_code + |1: + | EXT_CALL zend_jit_symtable_find, r0 + | jmp >1 + |.code + | EXT_CALL zend_hash_find, r0 + |1: + } else { + | EXT_CALL _zend_hash_find_known_hash, r0 + } + | test r0, r0 + | jz >9 // NOT_FOUND + | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) + | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 + | GET_Z_PTR r0, r0 + |1: + break; + case BP_VAR_R: + case BP_VAR_IS: + case BP_VAR_UNSET: + if (opline->op2_type != IS_CONST) { + | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' + | jle >1 + |.cold_code + |1: + | EXT_CALL zend_jit_symtable_find, r0 + | jmp >1 + |.code + | EXT_CALL zend_hash_find, r0 + |1: + } else { + | EXT_CALL _zend_hash_find_known_hash, r0 + } + | test r0, r0 + | jz >2 // NOT_FOUND + | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) + | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW + |.cold_code + |1: + | // retval = Z_INDIRECT_P(retval); + | GET_Z_PTR r0, r0 + | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 + |2: + switch (type) { + case BP_VAR_R: + // zend_error(E_NOTICE, "Undefined index: %s", ZSTR_VAL(offset_key)); + | UNDEFINED_INDEX opline + | jmp >9 + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + | // retval = &EG(uninitialized_zval); + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL + | jmp >9 + break; + default: + ZEND_ASSERT(0); + } + |.code + break; + case BP_VAR_RW: + | SAVE_VALID_OPLINE opline + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_lookup_rw, r0 + } else { + | EXT_CALL zend_jit_hash_lookup_rw, r0 + } + break; + case BP_VAR_W: + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_lookup_w, r0 + } else { + | EXT_CALL zend_jit_hash_lookup_w, r0 + } + break; + default: + ZEND_ASSERT(0); + } + } + + if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { + |5: + if (op1_info & MAY_BE_ARRAY_OF_REF) { + | ZVAL_DEREF r0, MAY_BE_REF + } + | cmp byte [r0 + 8], IS_NULL + | jle >9 // NOT FOUND + } + + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.cold_code + |3: + } + | SAVE_VALID_OPLINE opline + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + switch (type) { + case BP_VAR_R: + |.if X64 + | LOAD_ZVAL_ADDR CARG3, res_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR res_addr, r0 + |.endif + | EXT_CALL zend_jit_fetch_dim_r_helper, r0 + |.if not(X64) + | add r4, 12 + |.endif + | jmp >9 + break; + case BP_JIT_IS: + | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 + | test r0, r0 + | jne >8 + | jmp >9 + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + |.if X64 + | LOAD_ZVAL_ADDR CARG3, res_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR res_addr, r0 + |.endif + | EXT_CALL zend_jit_fetch_dim_is_helper, r0 + |.if not(X64) + | add r4, 12 + |.endif + | jmp >9 + break; + case BP_VAR_RW: + | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 + | test r0, r0 + | jne >8 + | jmp >9 + break; + case BP_VAR_W: + | EXT_CALL zend_jit_fetch_dim_w_helper, r0 + | test r0, r0 + | jne >8 + | jmp >9 + break; + default: + ZEND_ASSERT(0); + } + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.code + } + } + + return 1; +} + +static int zend_jit_simple_assign(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_jit_addr var_addr, + uint32_t var_info, + zend_uchar val_type, + znode_op val, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr res_addr, + int in_cold) +/* Labels: 1,2,3 */ +{ + ZEND_ASSERT(Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0); + if (Z_MODE(val_addr) == IS_CONST_ZVAL) { + zval *zv = Z_ZV(val_addr); + + if (!res_addr) { + | ZVAL_COPY_CONST var_addr, var_info, zv, r0 + } else { + | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, zv, r0 + } + if (Z_REFCOUNTED_P(zv)) { + if (!res_addr) { + | ADDREF_CONST zv, r0 + } else { + | ADDREF_CONST_2 zv, r0 + } + } + } else { + if (val_info & MAY_BE_UNDEF) { + if (in_cold) { + | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 + } else { + | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 + |.cold_code + |1: + } + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + if (Z_REG(var_addr) != ZREG_FP) { + | mov aword T1, Ra(Z_REG(var_addr)) // save + } + | SAVE_VALID_OPLINE opline + | mov FCARG1d, val.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (Z_REG(var_addr) != ZREG_FP) { + | mov Ra(Z_REG(var_addr)), aword T1 // restore + } + | SET_ZVAL_TYPE_INFO var_addr, IS_NULL + if (res_addr) { + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL + } + | jmp >3 + if (in_cold) { + |2: + } else { + |.code + } + } + if (val_info & MAY_BE_REF) { + if (val_type == IS_CV) { + ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); + | LOAD_ZVAL_ADDR r2, val_addr + | ZVAL_DEREF r2, val_info + val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); + } else { + zend_jit_addr ref_addr; + + if (in_cold) { + | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 + } else { + | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 + |.cold_code + |1: + } + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR r2, val_addr + | GC_DELREF r2 + | // ZVAL_COPY_VALUE(return_value, &ref->value); + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); + if (!res_addr) { + | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_R2, ZREG_R0 + } else { + | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_R2, ZREG_R0 + } + | je >2 + | IF_NOT_REFCOUNTED dh, >3 + if (!res_addr) { + | GC_ADDREF r0 + } else { + | add dword [r0], 2 + } + | jmp >3 + |2: + if (res_addr) { + | IF_NOT_REFCOUNTED dh, >2 + | GC_ADDREF r0 + |2: + } + | EFREE_24 aword [Ra(Z_REG(val_addr))+Z_OFFSET(val_addr)], op_array, opline + | jmp >3 + if (in_cold) { + |1: + } else { + |.code + } + } + } + + if (!res_addr) { + | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, ZREG_R0 + } else { + | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, ZREG_R0 + } + + if (val_type == IS_CV) { + if (!res_addr) { + | TRY_ADDREF val_info, dh, r0 + } else { + | TRY_ADDREF_2 val_info, dh, r0 + } + } else { + if (res_addr) { + | TRY_ADDREF val_info, dh, r0 + } + } + |3: + } + return 1; +} + +static int zend_jit_assign_to_variable(dasm_State **Dst, + const zend_op *opline, + zend_op_array *op_array, + zend_ssa *ssa, + zend_jit_addr var_addr, + uint32_t var_info, + zend_uchar val_type, + znode_op val, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr res_addr) +/* Labels: 1,2,3,4,5,8 */ +{ + //ZEND_ASSERT(Z_MODE(var_addr) == IS_MEM_ZVAL); + if (var_info & MAY_BE_REF) { + if (Z_MODE(var_addr) != IS_REG || Z_REG(var_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, var_addr + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + | // if (Z_ISREF_P(variable_ptr)) { + | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 + | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { + | GET_Z_PTR FCARG1a, FCARG1a + | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 + | jnz >2 + | add FCARG1a, offsetof(zend_reference, val) + |.cold_code + |2: + | LOAD_ZVAL_ADDR FCARG2a, val_addr + if (val_type == IS_CONST) { + | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 + } else if (val_type == IS_TMP_VAR) { + | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 + } else if (val_type == IS_VAR) { + | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 + } else if (val_type == IS_CV) { + | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 + } else { + ZEND_ASSERT(0); + } + | jmp >8 + |.code + |1: + } + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + int in_cold = 0; + + ZEND_ASSERT(Z_REG(var_addr) != ZREG_R0); + if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_ZVAL_REFCOUNTED var_addr, >1 + |.cold_code + |1: + in_cold = 1; + } + | // TODO: support for object->set + | // TODO: support for assignment to itself + | GET_ZVAL_PTR r0, var_addr + | GC_DELREF r0 + if (RC_MAY_BE_1(var_info)) { + if (RC_MAY_BE_N(var_info)) { + | jnz >4 + } + | mov aword T1, r0 // save + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, val_type, val, val_addr, val_info, res_addr, in_cold)) { + return 0; + } + | mov FCARG1a, aword T1 // restore + if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + | cmp dword [FCARG1a], 0 + | jnz >8 + } + | ZVAL_DTOR_FUNC var_info, opline + | jmp >8 + |4: + } + if (RC_MAY_BE_N(var_info)) { + if (Z_REG(var_addr) == ZREG_FP) { + | GET_ZVAL_PTR FCARG1a, var_addr + | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 + } else if (Z_REG(var_addr) != ZREG_FCARG1a) { + | GET_ZVAL_PTR FCARG1a, var_addr + | IF_GC_MAY_NOT_LEAK FCARG1a, eax, >5 + | mov T1, Ra(Z_REG(var_addr)) // save + } else { + | GET_ZVAL_PTR r0, var_addr + | IF_GC_MAY_NOT_LEAK r0, eax, >5 + | mov T1, Ra(Z_REG(var_addr)) // save + | GET_ZVAL_PTR FCARG1a, var_addr + } + | EXT_CALL gc_possible_root, r0 + if (Z_REG(var_addr) != ZREG_FP) { + | mov Ra(Z_REG(var_addr)), T1 // restore + } + if (in_cold) { + | jmp >5 + |.code + } + } else if (in_cold) { + ZEND_ASSERT(RC_MAY_BE_1(var_info)); + |.code + } + |5: + } + + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, val_type, val, val_addr, val_info, res_addr, 0)) { + return 0; + } + |8: + + return 1; +} + +static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info, val_info; + zend_jit_addr op1_addr, op2_addr, op3_addr, res_addr; + + if (opline->op1_type != IS_CV) { + goto fallback; + } + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + val_info = OP1_DATA_INFO(); + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline+1, NULL, -1); + if (opline->result_type == IS_UNUSED) { + res_addr = 0; + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + } + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 + } + | SEPARATE_ARRAY op1_addr, op1_info, 1 + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE + | jg >7 + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | mov T1, Ra(Z_REG(op1_addr)) // save + } + | EXT_CALL _zend_new_array_0, r0 + if (Z_REG(op1_addr) != ZREG_FP) { + | mov Ra(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX + | mov FCARG1a, r0 + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |6: + if (opline->op2_type == IS_UNUSED) { + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_next_index_insert, r0 + | // if (UNEXPECTED(!var_ptr)) { + | test r0, r0 + | jz >1 + |.cold_code + |1: + | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied"); + | CANNOT_ADD_ELEMENT opline + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | jmp >9 + |.code + | mov FCARG1a, r0 + } else { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_W, op1_info, op2_info, 8, 8)) { + return 0; + } + + |8: + | mov FCARG1a, r0 + } + + if (opline->op2_type == IS_UNUSED) { + uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr, 0)) { + return 0; + } + } else { + uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } + | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, val_info, res_addr)) { + return 0; + } + } + } + + if (((op1_info & MAY_BE_ARRAY) && + (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || + (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |.cold_code + |7: + } + + if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && + (op1_info & MAY_BE_ARRAY)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE + | jg >2 + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | mov T1, Ra(Z_REG(op1_addr)) // save + } + | EXT_CALL _zend_new_array_0, r0 + if (Z_REG(op1_addr) != ZREG_FP) { + | mov Ra(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX + | mov FCARG1a, r0 + | // ZEND_VM_C_GOTO(assign_dim_op_new_array); + | jmp <6 + |2: + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | SAVE_VALID_OPLINE opline + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + if (opline->op2_type == IS_UNUSED) { + | xor FCARG2a, FCARG2a + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } + |.if not(X64) + | sub r4, 8 + |.endif + if (opline->result_type == IS_UNUSED) { + |.if X64 + | xor CARG4, CARG4 + |.else + | push 0 + |.endif + } else { + zend_jit_addr res_addr; + + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + |.if X64 + | LOAD_ZVAL_ADDR CARG4, res_addr + |.else + | PUSH_ZVAL_ADDR res_addr, r0 + |.endif + } + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op3_addr + |.else + | PUSH_ZVAL_ADDR op3_addr, r0 + |.endif + | EXT_CALL zend_jit_assign_dim_helper, r0 + |.if not(X64) + | add r4, 8 + |.endif + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { + /* ASSIGN_DIM may increase refcount of the value */ + val_info |= MAY_BE_RCN; + } +#endif + + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, op_array, opline + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | jmp >9 // END + } + |.code + } + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { + /* ASSIGN_DIM may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + |9: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } + + return 1; + +fallback: + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, op3_addr, var_addr; + int op3_ssa_var; + + if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + goto fallback; + } + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op3_ssa_var = ssa->ops[opline - op_array->opcodes + 1].op1_use; + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + op3_addr = zend_jit_decode_op(op_array, (opline+1)->op1_type, (opline+1)->op1, opline+1, NULL, -1); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 + } + | SEPARATE_ARRAY op1_addr, op1_info, 1 + } + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE + | jg >7 + } + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 + } + | SAVE_VALID_OPLINE opline + | mov FCARG1a, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | mov T1, Ra(Z_REG(op1_addr)) // save + } + | EXT_CALL _zend_new_array_0, r0 + if (Z_REG(op1_addr) != ZREG_FP) { + | mov Ra(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX + | mov FCARG1a, r0 + if (op1_info & MAY_BE_ARRAY) { + | jmp >1 + |.code + |1: + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + uint32_t var_info = zend_array_element_type(op1_info, 0, 0); + uint32_t var_def_info = zend_array_element_type(OP1_DEF_INFO(), 1, 0); + + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } + |6: + if (opline->op2_type == IS_UNUSED) { + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_next_index_insert, r0 + | // if (UNEXPECTED(!var_ptr)) { + | test r0, r0 + | jz >1 + |.cold_code + |1: + | // zend_error(E_WARNING, "Cannot add element to the array as the next element is already occupied"); + | CANNOT_ADD_ELEMENT opline + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | jmp >9 + |.code + | mov FCARG1a, r0 + } else { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_VAR_RW, op1_info, op2_info, 8, 8)) { + return 0; + } + + |8: + | mov FCARG1a, r0 + if (op1_info & (MAY_BE_ARRAY_OF_REF)) { + | ZVAL_DEREF FCARG1a, MAY_BE_REF + } + } + + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + if (!zend_jit_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), 0, var_addr, var_def_info, var_info)) { + return 0; + } + break; + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_MOD: + if (!zend_jit_long_math_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, -1, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), op3_ssa_var, 0, var_addr, OP1_DEF_INFO(), var_info)) { + return 0; + } + break; + case ZEND_ASSIGN_CONCAT: + if (!zend_jit_concat_helper(Dst, opline, op_array, ssa, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, OP1_DATA_INFO(), var_addr, OP1_DEF_INFO())) { + return 0; + } + break; + default: + ZEND_ASSERT(0); + } + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + binary_op_type binary_op; + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |.cold_code + |7: + } + + | SAVE_VALID_OPLINE opline + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + if (opline->op2_type == IS_UNUSED) { + | xor FCARG2a, FCARG2a + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } + binary_op = get_binary_op(opline->opcode); + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op3_addr + | LOAD_ADDR CARG4, binary_op + |.else + | sub r4, 8 + | PUSH_ADDR binary_op, r0 + | PUSH_ZVAL_ADDR op3_addr, r0 + |.endif + | EXT_CALL zend_jit_assign_dim_op_helper, r0 + |.if not(X64) + | add r4, 8 + |.endif + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + | jmp >9 // END + |.code + } + } + + |9: + + return 1; + +fallback: + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr; + int op1_ssa_var, op2_ssa_var; + + if (opline->extended_value == ZEND_ASSIGN_DIM) { + return zend_jit_assign_dim_op(Dst, opline, op_array, ssa); + } else if (opline->extended_value == ZEND_ASSIGN_OBJ || opline->extended_value == ZEND_ASSIGN_STATIC_PROP) { + goto fallback; + } + + if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { + goto fallback; + } + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_ssa_var = ssa->ops[opline - op_array->opcodes].op1_use; + op2_ssa_var = ssa->ops[opline - op_array->opcodes].op2_use; + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + goto fallback; + } + + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || + !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + goto fallback; + } + break; + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_MOD: + if (!(op1_info & MAY_BE_LONG) || + !(op2_info & MAY_BE_LONG)) { + goto fallback; + } + break; + case ZEND_ASSIGN_CONCAT: + if (!(op1_info & MAY_BE_STRING) || + !(op2_info & MAY_BE_STRING)) { + goto fallback; + } + break; + default: + ZEND_ASSERT(0); + } + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + + switch (opline->opcode) { + case ZEND_ASSIGN_ADD: + case ZEND_ASSIGN_SUB: + case ZEND_ASSIGN_MUL: + case ZEND_ASSIGN_DIV: + return zend_jit_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info); + case ZEND_ASSIGN_BW_OR: + case ZEND_ASSIGN_BW_AND: + case ZEND_ASSIGN_BW_XOR: + case ZEND_ASSIGN_SL: + case ZEND_ASSIGN_SR: + case ZEND_ASSIGN_MOD: + return zend_jit_long_math_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, op1_ssa_var, opline->op2_type, opline->op2, op2_addr, op2_info, op2_ssa_var, opline->op1.var, op1_addr, OP1_DEF_INFO(), op1_info); + case ZEND_ASSIGN_CONCAT: + return zend_jit_concat_helper(Dst, opline, op_array, ssa, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, OP1_DEF_INFO()); + default: + ZEND_ASSERT(0); + } + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) +{ + unsigned int target_label; + zend_bool swap = 0; + + if (Z_MODE(op1_addr) == IS_REG) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) + } else { + | LONG_OP cmp, Z_REG(op1_addr), op2_addr + } + } else if (Z_MODE(op2_addr) == IS_REG) { + if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { + | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) + } else { + | LONG_OP cmp, Z_REG(op2_addr), op1_addr + } + swap = 1; + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { + | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) + swap = 1; + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { + | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) + } else { + | GET_ZVAL_LVAL ZREG_R0, op1_addr + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + | test r0, r0 + } else { + | LONG_OP cmp, ZREG_R0, op2_addr + } + } + if (((opline+1)->opcode == ZEND_JMPZ_EX || + (opline+1)->opcode == ZEND_JMPNZ_EX) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | sete al + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | setne al + break; + case ZEND_IS_SMALLER: + if (swap) { + | setg al + } else { + | setl al + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | setge al + } else { + | setle al + } + break; + default: + ZEND_ASSERT(0); + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + } + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPZ_EX) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | jle => target_label + } else { + | jge => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | jl => target_label + } else { + | jg => target_label + } + break; + default: + ZEND_ASSERT(0); + } + } else if (((opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPNZ_EX) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | je => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | jne => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | jg => target_label + } else { + | jl => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | jge => target_label + } else { + | jle => target_label + } + break; + default: + ZEND_ASSERT(0); + } + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | jle => target_label + } else { + | jge => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | jl => target_label + } else { + | jg => target_label + } + break; + default: + ZEND_ASSERT(0); + } + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp => target_label + } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | sete al + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | setne al + break; + case ZEND_IS_SMALLER: + if (swap) { + | setg al + } else { + | setl al + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | setge al + } else { + | setle al + } + break; + default: + ZEND_ASSERT(0); + } + | movzx eax, al + | add eax, 2 + | SET_ZVAL_TYPE_INFO res_addr, eax + } + + return 1; +} + +static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_bool swap) +{ + unsigned int target_label; + + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | jne => target_label + | jp => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | jp >1 + | je => target_label + |1: + break; + case ZEND_IS_SMALLER: + if (swap) { + | jbe => target_label + } else { + | jae => target_label + | jp => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | jb => target_label + } else { + | ja => target_label + | jp => target_label + } + break; + default: + ZEND_ASSERT(0); + } + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | jp >1 + | je => target_label + |1: + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | jne => target_label + | jp => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | ja => target_label + } else { + | jp >1 + | jb => target_label + |1: + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | jae => target_label + } else { + | jp >1 + | jbe => target_label + |1: + } + break; + default: + ZEND_ASSERT(0); + } + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + unsigned int target_label2 = ssa->cfg.blocks[b].successors[1]; + + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | jne => target_label + | jp => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | jp => target_label2 + | je => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | jbe => target_label + } else { + | jae => target_label + | jp => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | jb => target_label + } else { + | ja => target_label + | jp => target_label + } + break; + default: + ZEND_ASSERT(0); + } + | jmp => target_label2 + } else if ((opline+1)->opcode == ZEND_JMPZ_EX && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jne => target_label + | jp => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | je => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + break; + case ZEND_IS_SMALLER: + if (swap) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jbe => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jae => target_label + | jp => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jb => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | ja => target_label + | jp => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } + break; + default: + ZEND_ASSERT(0); + } + } else if ((opline+1)->opcode == ZEND_JMPNZ_EX && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | je => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jne => target_label + | jp => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + break; + case ZEND_IS_SMALLER: + if (swap) { + | seta al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | ja => target_label + } else { + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jb => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | setae al + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + | jae => target_label + } else { + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jbe => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } + break; + default: + ZEND_ASSERT(0); + } + } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + | jp >1 + | mov eax, IS_TRUE + | je >2 + |1: + | mov eax, IS_FALSE + |2: + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | jp >1 + | mov eax, IS_FALSE + | je >2 + |1: + | mov eax, IS_TRUE + |2: + break; + case ZEND_IS_SMALLER: + if (swap) { + | seta al + | movzx eax, al + | add eax, 2 + } else { + | jp >1 + | mov eax, IS_TRUE + | jb >2 + |1: + | mov eax, IS_FALSE + |2: + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | setae al + | movzx eax, al + | add eax, 2 + } else { + | jp >1 + | mov eax, IS_TRUE + | jbe >2 + |1: + | mov eax, IS_FALSE + |2: + } + break; + default: + ZEND_ASSERT(0); + } + | SET_ZVAL_TYPE_INFO res_addr, eax + } + + return 1; +} + +static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) +{ + |.if SSE + zend_reg tmp_reg = ZREG_XMM0; + + | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr + |.else + | FPU_GET_ZVAL_DVAL op2_addr + | FPU_GET_ZVAL_LVAL op1_addr + | fucomip st1 + | fstp st0 + |.endif + + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, 0); +} + +static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) +{ + zend_bool swap = 0; + + |.if SSE + zend_reg tmp_reg = ZREG_XMM0; + + | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr + swap = 1; + |.else + | FPU_GET_ZVAL_LVAL op2_addr + | FPU_GET_ZVAL_DVAL op1_addr + | fucomip st1 + | fstp st0 + |.endif + + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, swap); +} + +static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_jit_addr op1_addr, zend_jit_addr op2_addr) +{ + zend_bool swap = 0; + + |.if SSE + if (Z_MODE(op1_addr) == IS_REG) { + | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr + } else if (Z_MODE(op2_addr) == IS_REG) { + | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr + swap = 1; + } else { + zend_reg tmp_reg = ZREG_XMM0; + + | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr + | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr + } + |.else + | FPU_GET_ZVAL_DVAL op2_addr + | FPU_GET_ZVAL_DVAL op1_addr + | fucomip st1 + | fstp st0 + |.endif + + return zend_jit_cmp_double_common(Dst, opline, b, op_array, ssa, swap); +} + +static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa) +{ + unsigned int target_label; + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + | LONG_OP_WITH_CONST cmp, res_addr, Z_L(0) + if (((opline+1)->opcode == ZEND_JMPZ_EX || + (opline+1)->opcode == ZEND_JMPNZ_EX) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | sete al + break; + case ZEND_IS_NOT_EQUAL: + | setne al + break; + case ZEND_IS_SMALLER: + | setl al + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setle al + break; + default: + ZEND_ASSERT(0); + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + } + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPZ_EX) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + | jge => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jg => target_label + break; + default: + ZEND_ASSERT(0); + } + } else if (((opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPNZ_EX) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | je => target_label + break; + case ZEND_IS_NOT_EQUAL: + | jne => target_label + break; + case ZEND_IS_SMALLER: + | jl => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jle => target_label + break; + default: + ZEND_ASSERT(0); + } + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | jne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | je => target_label + break; + case ZEND_IS_SMALLER: + | jge => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | jg => target_label + break; + default: + ZEND_ASSERT(0); + } + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp => target_label + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | sete al + break; + case ZEND_IS_NOT_EQUAL: + | setne al + break; + case ZEND_IS_SMALLER: + | setl al + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | setle al + break; + default: + ZEND_ASSERT(0); + } + | movzx eax, al + | add eax, 2 + | SET_ZVAL_TYPE_INFO res_addr, eax + } + + return 1; +} + +static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info, op2_info; + zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + zend_bool has_slow; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + has_slow = + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); + + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 + } + } + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { + if (op2_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 + |.cold_code + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 + } + if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + | jmp >6 + |.code + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 + } + } + if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + |4: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 + } + } + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + | jmp >6 + } + if (!same_ops) { + |5: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 + } + if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + | jmp >6 + } + |.code + } + } else if ((op1_info & MAY_BE_DOUBLE) && + !(op1_info & MAY_BE_LONG) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op2_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 + } + } + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + } + if (!same_ops && (op2_info & MAY_BE_LONG)) { + if (op2_info & MAY_BE_DOUBLE) { + |.cold_code + } + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 + } + if (!zend_jit_cmp_double_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + if (op2_info & MAY_BE_DOUBLE) { + | jmp >6 + |.code + } + } + } else if ((op2_info & MAY_BE_DOUBLE) && + !(op2_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 + } + if (op1_info & MAY_BE_DOUBLE) { + if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op1_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 + } + } + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + } + if (!same_ops && (op1_info & MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + } + |3: + if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 + } + if (!zend_jit_cmp_long_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + | jmp >6 + |.code + } + } + } + + if (has_slow || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + if (has_slow) { + |.cold_code + |9: + } + | SAVE_VALID_OPLINE opline + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2a, op1_addr + if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { + | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1 + | mov FCARG1a, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval + |1: + } + if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 + | mov T1, FCARG2a // save + | mov FCARG1a, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | mov FCARG2a, T1 // restore + |.if X64 + | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval + |.else + | sub r4, 12 + | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 + |.endif + | jmp >2 + |1: + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif + |2: + } else { + |.if X64 + | LOAD_ZVAL_ADDR CARG3, op2_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR op2_addr, r0 + |.endif + } + | LOAD_ZVAL_ADDR FCARG1a, res_addr + | EXT_CALL compare_function, r0 + |.if not(X64) + | add r4, 12 + |.endif + if (opline->opcode != ZEND_CASE) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline + } + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (!zend_jit_cmp_slow(Dst, opline, b, op_array, ssa)) { + return 0; + } + if (has_slow) { + | jmp >6 + |.code + } + } + + |6: + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZ_EX || + (opline+1)->opcode == ZEND_JMPNZ_EX || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + } + + return 1; +} + +static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + zend_bool smart_branch = 0; + uint32_t identical_label = (uint32_t)-1; + uint32_t not_identical_label = (uint32_t)-1; + uint32_t op1_info, op2_info; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + zend_jit_addr op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + smart_branch = 1; + } + + if (smart_branch) { + if (opline->opcode == ZEND_IS_IDENTICAL) { + if ((opline+1)->opcode == ZEND_JMPZ) { + not_identical_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + identical_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + not_identical_label = ssa->cfg.blocks[b].successors[0]; + identical_label = ssa->cfg.blocks[b].successors[1]; + } else { + ZEND_ASSERT(0); + } + } else if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { + if ((opline+1)->opcode == ZEND_JMPZ) { + identical_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + not_identical_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + identical_label = ssa->cfg.blocks[b].successors[0]; + not_identical_label = ssa->cfg.blocks[b].successors[1]; + } else { + ZEND_ASSERT(0); + } + } else { + ZEND_ASSERT(0); + } + } + + if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { + op1_info |= MAY_BE_NULL; + op2_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval + | jmp >1 + |.code + |1: + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov aword T1, FCARG1a // save + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | mov FCARG1a, aword T1 // restore + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval + | jmp >1 + |.code + |1: + } else if (op1_info & MAY_BE_UNDEF) { + op1_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval + | jmp >1 + |.code + |1: + if (opline->op2_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } + } else if (op2_info & MAY_BE_UNDEF) { + op2_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 + |.cold_code + |1: + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval + | jmp >1 + |.code + |1: + if (opline->op1_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + } else { + if (opline->op1_type != IS_CONST) { + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + if (opline->op2_type != IS_CONST) { + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } + } + if (opline->op1_type & (IS_CV|IS_VAR)) { + | ZVAL_DEREF FCARG1a, op1_info + } + if (opline->op2_type & (IS_CV|IS_VAR)) { + | ZVAL_DEREF FCARG2a, op2_info + } + + if ((op1_info & op2_info & MAY_BE_ANY) == 0) { + if (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline + } + if (smart_branch) { + zend_jit_check_exception_undef_result(Dst, opline); + if (not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) + zend_jit_check_exception(Dst); + } + } else if (has_concrete_type(op1_info) && + has_concrete_type(op2_info) && + concrete_type(op1_info) == concrete_type(op2_info) && + concrete_type(op1_info) <= IS_TRUE) { + if (smart_branch) { + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE) + } + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { + if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { + if (smart_branch) { + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_TRUE : IS_FALSE) + } + } else { + if (smart_branch) { + if (not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode == ZEND_IS_IDENTICAL ? IS_FALSE : IS_TRUE) + } + } + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { + zval *val = Z_ZV(op1_addr); + + | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) + if (smart_branch) { + if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { + | jne >8 + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } else { + | jmp >9 + } + |8: + } else if (identical_label != (uint32_t)-1) { + | je =>identical_label + } else { + | je >9 + } + } else { + if (opline->opcode == ZEND_IS_IDENTICAL) { + | sete al + } else { + | setne al + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + } + if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); + } + if (smart_branch && not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { + zval *val = Z_ZV(op2_addr); + + | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) + if (smart_branch) { + if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { + | jne >8 + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } else { + | jmp >9 + } + |8: + } else if (identical_label != (uint32_t)-1) { + | je =>identical_label + } else { + | je >9 + } + } else { + if (opline->opcode == ZEND_IS_IDENTICAL) { + | sete al + } else { + | setne al + } + | movzx eax, al + | lea eax, [eax + 2] + | SET_ZVAL_TYPE_INFO res_addr, eax + } + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); + } + if (smart_branch && not_identical_label != (uint32_t)-1) { + | jmp =>not_identical_label + } + } else if ((op1_info & MAY_BE_ANY) == MAY_BE_LONG && + (op2_info & MAY_BE_ANY) == MAY_BE_LONG) { + if (!zend_jit_cmp_long_long(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + } else if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE && + (op2_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + if (!zend_jit_cmp_double_double(Dst, opline, b, op_array, ssa, op1_addr, op2_addr)) { + return 0; + } + } else { + if (opline->op1_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + if (opline->op2_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } + | EXT_CALL zend_is_identical, r0 + if (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { + | mov aword T1, r0 // save + | SAVE_VALID_OPLINE opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, op_array, opline + zend_jit_check_exception_undef_result(Dst, opline); + | mov r0, aword T1 // restore + } + if (smart_branch) { + | test al, al + if (not_identical_label != (uint32_t)-1) { + | jz =>not_identical_label + if (identical_label != (uint32_t)-1) { + | jmp =>identical_label + } + } else if (identical_label != (uint32_t)-1) { + | jnz =>identical_label + } + } else { + | movzx eax, al + if (opline->opcode == ZEND_IS_IDENTICAL) { + | lea eax, [eax + 2] + } else { + | neg eax + | lea eax, [eax + 3] + } + | SET_ZVAL_TYPE_INFO res_addr, eax + } + } + + |9: + + return 1; +} + +static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info = OP1_INFO(); + uint32_t true_label = -1; + uint32_t false_label = -1; + zend_bool set_bool = 0; + zend_bool set_bool_not = 0; + zend_bool jmp_done = 0; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + zend_jit_addr res_addr = 0; + + if (opline->opcode == ZEND_JMPZ) { + false_label = ssa->cfg.blocks[b].successors[0]; + } else if (opline->opcode == ZEND_JMPNZ) { + true_label = ssa->cfg.blocks[b].successors[0]; + } else if (opline->opcode == ZEND_JMPZNZ) { + true_label = ssa->cfg.blocks[b].successors[1]; + false_label = ssa->cfg.blocks[b].successors[0]; + } else if (opline->opcode == ZEND_BOOL) { + set_bool = 1; + } else if (opline->opcode == ZEND_BOOL_NOT) { + set_bool = 1; + set_bool_not = 1; + } else if (opline->opcode == ZEND_JMPZ_EX) { + set_bool = 1; + false_label = ssa->cfg.blocks[b].successors[0]; + } else if (opline->opcode == ZEND_JMPNZ_EX) { + set_bool = 1; + true_label = ssa->cfg.blocks[b].successors[0]; + } else { + ZEND_ASSERT(0); + } + + if (set_bool) { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); + } + + if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { + if (zend_is_true(Z_ZV(op1_addr))) { + /* Always TRUE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } + } + if (true_label != (uint32_t)-1) { + | jmp =>true_label; + } + } else { + /* Always FALSE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } + } + if (false_label != (uint32_t)-1) { + | jmp =>false_label; + } + } + return 1; + } + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { + if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { + /* Always TRUE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } + } + if (true_label != (uint32_t)-1) { + | jmp =>true_label; + } + } else { + if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { + /* Always FALSE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } + } + } else { + | CMP_ZVAL_TYPE op1_addr, IS_TRUE + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { + if ((op1_info & MAY_BE_LONG) && + !(op1_info & MAY_BE_UNDEF) && + !set_bool) { + if (false_label != (uint32_t)-1) { + | jl =>false_label + } else { + | jl >9 + } + jmp_done = 1; + } else { + | jg >2 + } + } + if (!(op1_info & MAY_BE_TRUE)) { + /* It's FALSE */ + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } + } + } else { + if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (set_bool) { + | jne >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + if (true_label != (uint32_t)-1) { + | jmp =>true_label + } else { + | jmp >9 + } + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } else { + if (true_label != (uint32_t)-1) { + | je =>true_label + } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | jne =>false_label + jmp_done = 1; + } else { + | je >9 + } + } + } else if (set_bool) { + | sete al + | movzx eax, al + if (set_bool_not) { + | neg eax + | add eax, 3 + } else { + | add eax, 2 + } + | SET_ZVAL_TYPE_INFO res_addr, eax + } + } + } + + /* It's FALSE, but may be UNDEF */ + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & MAY_BE_ANY) { + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 + |.cold_code + |1: + } + | mov FCARG1d, opline->op1.var + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_jit_undefined_op_helper, r0 + + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } + if (op1_info & MAY_BE_ANY) { + if (false_label == (uint32_t)-1) { + | jmp >9 + } + |.code + } + } + + if (!jmp_done) { + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } else if (op1_info & MAY_BE_LONG) { + | jmp >9 + } + } + } + } + + if (op1_info & MAY_BE_LONG) { + |2: + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 + } + if (Z_MODE(op1_addr) == IS_REG) { + | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) + } else { + | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) + } + if (set_bool) { + | setne al + | movzx eax, al + if (set_bool_not) { + | neg eax + | add eax, 3 + } else { + | lea eax, [eax + 2] + } + | SET_ZVAL_TYPE_INFO res_addr, eax + } + if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } + } else { + | je =>false_label + } + } + } + + if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + |.if SSE + if (zend_jit_x86_flags & ZEND_JIT_CPU_AVX) { + | vxorps xmm0, xmm0, xmm0 + } else { + | xorps xmm0, xmm0 + } + | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr + |.else + | FPU_GET_ZVAL_DVAL op1_addr + | fldz + | fucomip st1 + | fstp st0 + |.endif + + if (set_bool) { + if (false_label != (uint32_t)-1) { // JMPZ_EX + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jp >1 + | je => false_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + |1: + } else if (true_label != (uint32_t)-1) { // JMPNZ_EX + | jp >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jne => true_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } else if (set_bool_not) { // BOOL_NOT + | jp >1 + | mov eax, IS_TRUE + | je >2 + |1: + | mov eax, IS_FALSE + |2: + | SET_ZVAL_TYPE_INFO res_addr, eax + } else { // BOOL + | jp >1 + | mov eax, IS_TRUE + | jne >2 + |1: + | mov eax, IS_FALSE + |2: + | SET_ZVAL_TYPE_INFO res_addr, eax + } + } else { + ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); + if (false_label != (uint32_t)-1) { + | jp =>false_label + } else { + | jp >1 + } + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } + } else { + | je =>false_label + } + |1: + } + } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.cold_code + |2: + } + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + | SAVE_VALID_OPLINE opline + | EXT_CALL zend_is_true, r0 + + if (set_bool) { + if (set_bool_not) { + | neg eax + | add eax, 3 + } else { + | add eax, 2 + } + | SET_ZVAL_TYPE_INFO res_addr, eax + | FREE_OP opline->op1_type, opline->op1, op1_info, !(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)), op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + | CMP_ZVAL_TYPE res_addr, IS_FALSE + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + } + } else { + | je =>false_label + } + } + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + |.code + } + } else { + + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 + | GET_ZVAL_PTR FCARG1a, op1_addr + | GC_DELREF FCARG1a + | jnz >3 + | mov aword T1, r0 // save + | ZVAL_DTOR_FUNC op1_info, opline + | mov r0, aword T1 // restore + |3: + } + if (zend_may_throw(opline, op_array, ssa)) { + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 + | jne ->exception_handler_undef + } + + | test r0, r0 + if (true_label != (uint32_t)-1) { + | jne =>true_label + if (false_label != (uint32_t)-1) { + | jmp =>false_label + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + } + } else { + | je =>false_label + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | jmp >9 + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.code + } + } + } + + |9: + + return 1; +} + +static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info = OP1_INFO(); + zend_jit_addr op1_addr, res_addr; + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); + + if (ra + && ssa->ops[opline - op_array->opcodes].op1_def >= 0 + && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) { + zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def); + + if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { + return 0; + } + } + + if (!zend_jit_simple_assign(Dst, opline, op_array, ssa, res_addr, -1, opline->op1_type, opline->op1, op1_addr, op1_info, 0, 0)) { + return 0; + } + if (ra && !zend_jit_store_ssa_var_if_necessary(Dst, ssa, ra, res_addr, ssa->ops[opline - op_array->opcodes].result_def, ssa->ops[opline - op_array->opcodes].result_use)) { + return 0; + } + return 1; +} + +static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, res_addr; + + if (opline->op1_type != IS_CV || !ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_def : -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op2_use : -1); + if (opline->result_type == IS_UNUSED) { + res_addr = 0; + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].result_def : -1); + } + + if (ra + && ssa->ops[opline - op_array->opcodes].op2_def >= 0 + && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) { + zend_jit_addr op2_def_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, ra, ssa->ops[opline - op_array->opcodes].op2_def); + + if (!zend_jit_update_regs(Dst, op2_addr, op2_def_addr, op2_info, ra[ssa->ops[opline - op_array->opcodes].op2_use])) { + return 0; + } + } + + if (!zend_jit_assign_to_variable(Dst, opline, op_array, ssa, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr)) { + return 0; + } + + if (zend_may_throw(opline, op_array, ssa)) { + zend_jit_check_exception(Dst); + } + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_function *func) +{ + uint32_t used_stack; + + if (func) { + used_stack = zend_vm_calc_used_stack(opline->extended_value, func); + } else { + used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); + + | // if (EXPECTED(ZEND_USER_CODE(func->type))) { + | test byte [r0 + offsetof(zend_function, type)], 1 + | mov FCARG1a, used_stack + | jnz >1 + | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); + | mov edx, opline->extended_value + | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] + | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] + | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] + | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] + | shl edx, 5 + |.if X64 + | movsxd r2, edx + |.endif + | sub FCARG1a, r2 + |1: + } + + zend_jit_start_reuse_ip(); + + | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { + | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX + | // Check Stack Overflow + | MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2 + | sub r2, RX + if (func) { + | cmp r2, used_stack + } else { + | cmp r2, FCARG1a + } + | jb >1 + | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); + |.cold_code + |1: + if (func) { + | mov FCARG1d, used_stack + } +#ifdef _WIN32 + if (0) { +#else + if (func && func->type == ZEND_INTERNAL_FUNCTION) { +#endif + | EXT_CALL zend_jit_int_extend_stack_helper, r0 + } else { + | mov FCARG2a, r0 + | EXT_CALL zend_jit_extend_stack_helper, r0 + } + | mov RX, r0 + | jmp >1 + |.code + + if (func) { + | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 + } else { + | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 + } + | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); + | // ZEND_SET_CALL_INFO(call, 0, call_info); + | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) + | // call->func = func; +#ifdef _WIN32 + if (0) { +#else + if (func && func->type == ZEND_INTERNAL_FUNCTION) { +#endif + |1: + | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 + } else { + | mov aword EX:RX->func, r0 + |1: + } + | // Z_CE(call->This) = called_scope; + | mov aword EX:RX->This.value.ptr, 0 + | // ZEND_CALL_NUM_ARGS(call) = num_args; + | mov dword EX:RX->This.u2.num_args, opline->extended_value + return 1; +} + +static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline) +{ + int skip; + + if (!call_info) { + const zend_op *end = op_array->opcodes + op_array->last; + + opline++; + skip = 1; + while (opline != end) { + if (!skip) { + if (zend_may_throw(opline, op_array, ssa)) { + return 1; + } + } + switch (opline->opcode) { + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + skip = 0; + break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + case ZEND_INIT_FCALL: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_NEW: + case ZEND_INIT_USER_CALL: + case ZEND_FAST_CALL: + case ZEND_JMP: + case ZEND_JMPZNZ: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_ASSERT_CHECK: + case ZEND_CATCH: + case ZEND_DECLARE_ANON_CLASS: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + return 1; + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: + end = opline; + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + return 0; + } + opline++; + } + + return 1; + } else { + const zend_op *end = call_info->caller_call_opline; + + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + + opline++; + skip = 1; + while (opline != end) { + if (skip) { + switch (opline->opcode) { + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + skip = 0; + break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + return 1; + } + } else { + if (zend_may_throw(opline, op_array, ssa)) { + return 1; + } + } + opline++; + } + + return 0; + } +} + +static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, zend_op_array *op_array, zend_ssa *ssa, int call_level) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + zend_function *func = NULL; + + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_init_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func) { + func = call_info->callee_func; + } + } + +#ifdef _WIN32 + if (0) { +#else + if (func && func->type == ZEND_INTERNAL_FUNCTION) { +#endif + /* load constant address later */ + } else if (func && op_array == &func->op_array) { + /* recursive call */ + | mov r0, EX->func + } else { + | // if (CACHED_PTR(opline->result.num)) + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + opline->result.num] + | test r0, r0 + | jz >1 + |.cold_code + |1: + if (func && func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { + | LOAD_ADDR FCARG1a, func + | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 + | mov r1, EX->run_time_cache + | mov aword [r1 + opline->result.num], r0 + | jmp >3 + } else { + zval *zv = RT_CONSTANT(opline, opline->op2); + + if (opline->opcode == ZEND_INIT_FCALL) { + | LOAD_ADDR FCARG1a, Z_STR_P(zv); + } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { + | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1); + } else { + ZEND_ASSERT(0); + } + | EXT_CALL zend_jit_find_func_helper, r0 + | // CACHE_PTR(opline->result.num, fbc); + | mov r1, EX->run_time_cache + | mov aword [r1 + opline->result.num], r0 + | test r0, r0 + | jnz >3 + | // SAVE_OPLINE(); + | SAVE_VALID_OPLINE opline + | jmp ->undefined_function + } + |.code + |3: + } + + if (!zend_jit_push_call_frame(Dst, opline, op_array, func)) { + return 0; + } + + if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, opline)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } + } else { + delayed_call_chain = 1; + delayed_call_level = call_level; + } + + return 1; +} + +static uint32_t skip_valid_arguments(zend_op_array *op_array, zend_ssa *ssa, zend_call_info *call_info) +{ + uint32_t num_args = 0; + zend_function *func = call_info->callee_func; + + while (num_args < call_info->num_args) { + zend_arg_info *arg_info = func->op_array.arg_info + num_args; + + if (ZEND_TYPE_IS_SET(arg_info->type)) { + if (!ZEND_TYPE_IS_CLASS(arg_info->type)) { + unsigned char code = ZEND_TYPE_CODE(arg_info->type); + uint32_t info = _ssa_op1_info(op_array, ssa, call_info->arg_info[num_args].opline); + + if (code == _IS_BOOL) { + if (info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_FALSE|MAY_BE_TRUE))) { + break; + } + } else if (code <= IS_RESOURCE) { + if (info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (1 << code))) { + break; + } + } else { + break; + } + } else { + break; + } + } + num_args++; + } + return num_args; +} + +static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + zend_function *func = NULL; + uint32_t i; + zend_jit_addr res_addr; + + if (RETURN_VALUE_USED(opline)) { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + } else { +#ifdef _WIN64 + /* Reuse reserved arguments stack */ + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, 0x20); +#else + /* CPU stack alocated temorary zval */ + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, 8); +#endif + } + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_call_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func) { + func = call_info->callee_func; + } + } + if (!func) { + /* resolve function ar run time */ + } else if (func->type == ZEND_USER_FUNCTION) { + if (call_info->num_args > func->op_array.num_args || + (opline-1)->opcode == ZEND_SEND_UNPACK || + (opline-1)->opcode == ZEND_SEND_ARRAY) { + goto fallback; + } + } else if (func->type == ZEND_INTERNAL_FUNCTION) { +#if ZEND_DEBUG + if (func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { + goto fallback; + } +#endif + if ((opline-1)->opcode == ZEND_SEND_UNPACK || (opline-1)->opcode == ZEND_SEND_ARRAY) { + goto fallback; + } + } else { + ZEND_ASSERT(0); + } + + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + zend_jit_stop_reuse_ip(); + + | // fbc = call->func; + | // mov r2, EX:RX->func ??? + | // SAVE_OPLINE(); + | SAVE_VALID_OPLINE opline + + if (!delayed_call_chain) { + if (call_level == 1) { + | mov aword EX->call, 0 + } else { + | //EX(call) = call->prev_execute_data; + | mov r0, EX:RX->prev_execute_data + | mov EX->call, r0 + } + } + delayed_call_chain = 0; + + | //call->prev_execute_data = execute_data; + | mov EX:RX->prev_execute_data, EX + + if (!func) { + | mov r0, EX:RX->func + } + + if (opline->opcode == ZEND_DO_FCALL) { + if (!func) { + | test dword [r0 + offsetof(zend_op_array, fn_flags)], (ZEND_ACC_DEPRECATED|ZEND_ACC_ABSTRACT) + | jnz >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | jne ->exception_handler + | mov r0, EX:RX->func // reload + | jmp >1 + |.code + |1: + } else if (func->common.fn_flags & ZEND_ACC_ABSTRACT) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | jmp ->exception_handler + } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | jne ->exception_handler + | mov r0, EX:RX->func // reload + } + } + + if (!func) { + | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION + | jne >8 + } + + if (!func || func->type == ZEND_USER_FUNCTION) { + | // EX(call) = NULL; + | mov aword EX:RX->call, 0 + + if (RETURN_VALUE_USED(opline)) { + | // EX(return_value) = EX_VAR(opline->result.var); + | LOAD_ZVAL_ADDR r2, res_addr + | mov aword EX:RX->return_value, r2 + } else { + | // EX(return_value) = 0; + | mov aword EX:RX->return_value, 0 + } + + if (func) { + for (i = call_info->num_args; i < func->op_array.last_var; i++) { + uint32_t n = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | SET_Z_TYPE_INFO RX + n, IS_UNDEF + } + } + + //EX_LOAD_RUN_TIME_CACHE(op_array); + if (!func || func->op_array.cache_size) { + if (func && op_array == &func->op_array) { + /* recursive call */ + if (func->op_array.cache_size > sizeof(void*)) { + | mov r2, EX->run_time_cache + | mov EX:RX->run_time_cache, r2 + } + } else { + if (func) { + | mov r0, EX:RX->func + } + | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] +#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR + | mov r2, aword [r2] +#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET + | xor r1, r1 + | test r2, 1 + | jz >1 + | MEM_OP2_2_ZTS mov, r1, aword, compiler_globals, map_ptr_base, r1 + | sub r1, 1 + |1: + | mov r2, aword [r1 + r2] +#else +# error "Unknown ZEND_MAP_PTR_KIND" +#endif + | mov EX:RX->run_time_cache, r2 + } + } + + | // EG(current_execute_data) = execute_data; + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 + | mov FP, RX + + | // opline = op_array->opcodes; + if (func) { + uint32_t num_args; + + if (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + num_args = skip_valid_arguments(op_array, ssa, call_info); + } else { + num_args = call_info->num_args; + } + if (func && zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_IP_ADDR (func->op_array.opcodes + num_args) + } else { + if (func) { + | mov r0, EX->func + } + if (GCC_GLOBAL_REGS) { + | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + if (num_args) { + | add IP, (num_args * sizeof(zend_op)) + } + } else { + | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] + if (num_args) { + | add FCARG1a, (num_args * sizeof(zend_op)) + } + | mov aword EX->opline, FCARG1a + } + } + + if (op_array == &func->op_array) { + /* recursive call */ +#ifdef CONTEXT_THREADED_JIT + | call >1 + |.cold_code + |1: + | pop r0 + | jmp =>num_args + |.code +#else + | jmp =>num_args +#endif + return 1; + } + } else { + | // opline = op_array->opcodes + if (GCC_GLOBAL_REGS) { + | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] + } else { + | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] + | mov aword EX->opline, FCARG1a + } + | // first_extra_arg = op_array->num_args; + | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] + | // num_args = EX_NUM_ARGS(); + | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] + | // if (UNEXPECTED(num_args > first_extra_arg)) + | cmp edx, ecx + | jl >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, FP + } + | EXT_CALL zend_jit_copy_extra_args_helper, r0 + | mov r0, EX->func // reload + | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload + | jmp >1 + |.code + | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) + | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS + | jnz >1 + | // opline += num_args; + |.if X64 + | movsxd r2, ecx + | imul r2, r2, sizeof(zend_op) + |.else + | imul r2, ecx, sizeof(zend_op) + |.endif + | ADD_IP r2 + |1: + | // if (EXPECTED((int)num_args < op_array->last_var)) { + | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] + | sub edx, ecx + | jle >3 //??? + | // zval *var = EX_VAR_NUM(num_args); + |.if X64 + | movsxd r1, ecx + |.endif + | shl r1, 4 + | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] + |2: + | SET_Z_TYPE_INFO r1, IS_UNDEF + | sub edx, 1 + | lea r1, [r1 + 16] + | jne <2 + |3: + } + +#ifdef CONTEXT_THREADED_JIT + | call ->context_threaded_call + if (!func) { + | jmp >9 + } +#else + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD + | JMP_IP + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment + | JMP_IP + } else { + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 1 // ZEND_VM_ENTER + | ret + } +#endif + } + + if (!func || func->type == ZEND_INTERNAL_FUNCTION) { + if (!func) { + |8: + } + if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { + if (!func) { + | test dword [r0 + offsetof(zend_op_array, fn_flags)], (ZEND_ACC_DEPRECATED|ZEND_ACC_ABSTRACT) + | jnz >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | jne ->exception_handler + | mov r0, EX:RX->func // reload + | jmp >1 + |.code + |1: + } else if (func->common.fn_flags & ZEND_ACC_ABSTRACT) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | jmp ->exception_handler + } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + if (!GCC_GLOBAL_REGS) { + | mov FCARG1a, RX + } + | EXT_CALL zend_jit_deprecated_or_abstract_helper, r0 + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | jne ->exception_handler + | mov r0, EX:RX->func // reload + } + } + + if (!RETURN_VALUE_USED(opline)) { + |.if not(X64WIN) + | sub r4, 16 /* alloca() */ + |.endif + } + + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | LOAD_ZVAL_ADDR FCARG2a, res_addr + | SET_Z_TYPE_INFO FCARG2a, IS_NULL + + | // EG(current_execute_data) = execute_data; + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 + + zend_jit_reset_opline(Dst, NULL); + + | // fbc->internal_function.handler(call, ret); + | mov FCARG1a, RX + if (func) { + | EXT_CALL func->internal_function.handler, r0 + } else { + | call aword [r0 + offsetof(zend_internal_function, handler)] + } + + | // EG(current_execute_data) = execute_data; + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 + + | // zend_vm_stack_free_args(call); + if (func) { + for (i = 0; i < call_info->num_args; i++ ) { + uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, 1, opline + } + } else { + | mov FCARG1a, RX + | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 + } + + |8: + if (opline->opcode == ZEND_DO_FCALL) { + // TODO: optimize ??? + | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) + | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16) + | jnz >1 + |.cold_code + |1: + | GET_Z_PTR r0, RX + offsetof(zend_execute_data, This) + | // OBJ_RELEASE(object); + | OBJ_RELEASE r0, ecx, >2 + | jmp >2 + |.code + |2: + } + + | // zend_vm_stack_free_call_frame(call); + | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) + | jnz >1 + |.cold_code + |1: + | mov FCARG1a, RX + | EXT_CALL zend_jit_free_call_frame, r0 + | jmp >1 + |.code + | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0 + |1: + + if (!RETURN_VALUE_USED(opline)) { + uint32_t func_info = call_info ? + zend_get_func_info(call_info, ssa) : + (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); + + if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, 0, opline + } + |.if not(X64WIN) + | add r4, 16 /* revert alloca() */ + |.endif + } + + | // if (UNEXPECTED(EG(exception) != NULL)) { + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | jne >1 + |.cold_code + |1: + | LOAD_IP_ADDR opline + | jmp ->icall_throw_handler + |.code + + // TODO: Can we avoid checking for interrupts after each call ??? + if (!zend_jit_check_timeout(Dst, opline + 1)) { + return 0; + } + if (opline->opcode != ZEND_DO_ICALL) { + | LOAD_IP_ADDR (opline + 1) + } + } + + if (!func) { + |9: + } + + return 1; + +fallback: + /* fallback to subroutine threading */ + if (opline->opcode == ZEND_DO_FCALL || + opline->opcode == ZEND_DO_UCALL || + opline->opcode == ZEND_DO_FCALL_BY_NAME ){ + return zend_jit_call(Dst, opline, next_block); + } else { + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); + } +} + +static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info; + uint32_t arg_num = opline->op2.num; + zend_jit_addr op1_addr, arg_addr; + + if (opline->opcode == ZEND_SEND_VAL_EX && arg_num > MAX_ARG_FLAG_NUM) { + goto fallback; + } + + op1_info = OP1_INFO(); + + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + + if (opline->opcode == ZEND_SEND_VAL_EX) { + uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); + + | mov r0, EX:RX->func + if (arg_num <= MAX_ARG_FLAG_NUM) { + | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask + | jnz >1 + } else { + ZEND_ASSERT(0); + } + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | jmp ->throw_cannot_pass_by_ref + |.code + } + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + + | ZVAL_COPY_CONST arg_addr, -1, zv, r0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r0 + } + } else { + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + } + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, int cold) +{ + uint32_t op1_info; + zend_jit_addr op1_addr, arg_addr, ref_addr; + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + op1_info = OP1_INFO(); + + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + + if (opline->op1_type == IS_VAR) { + | LOAD_ZVAL_ADDR r0, op1_addr + | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { + | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 + | // ret = Z_INDIRECT_P(ret); + | GET_Z_PTR r0, r0 + |1: + if (op1_info & MAY_BE_ERROR) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + if (cold) { + | IF_NOT_Z_TYPE r0, _IS_ERROR, >1 + } else { + | IF_Z_TYPE r0, _IS_ERROR, >1 + |.cold_code + |1: + } + } + + | // ZVAL_NEW_EMPTY_REF(arg); + | EMALLOC sizeof(zend_reference), op_array, opline + | SET_ZVAL_PTR arg_addr, r0 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX + | mov dword [r0], 1 + | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], IS_REFERENCE + | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 8); + | // ZVAL_NULL(Z_REFVAL_P(arg)); + | SET_ZVAL_TYPE_INFO ref_addr, IS_NULL + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | jmp >7 + if (cold) { + |1: + } else { + |.code + } + } + } + } else if (opline->op1_type == IS_CV) { + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL + | jmp >2 + |1: + } + op1_info &= ~MAY_BE_UNDEF; + op1_info |= MAY_BE_NULL; + } + } else { + ZEND_ASSERT(0); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { + if (op1_info & MAY_BE_REF) { + if (opline->op1_type == IS_VAR) { + | IF_NOT_Z_TYPE r0, IS_REFERENCE, >2 + | GET_Z_PTR r1, r0 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 + | GET_ZVAL_PTR r1, op1_addr + } + | GC_ADDREF r1 + | SET_ZVAL_PTR arg_addr, r1 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX + | jmp >6 + } + |2: + | // ZVAL_NEW_REF(arg, varptr); + if (opline->op1_type == IS_VAR) { + | mov aword T1, r0 // save + } + | EMALLOC sizeof(zend_reference), op_array, opline + | mov dword [r0], 2 + | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], IS_REFERENCE + | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 8); + if (opline->op1_type == IS_VAR) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); + + | mov r1, aword T1 // restore + | ZVAL_COPY_VALUE ref_addr, -1, val_addr, op1_info, ZREG_R2, ZREG_R2 + | SET_ZVAL_PTR val_addr, r0 + | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX + } else { + | ZVAL_COPY_VALUE ref_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 + | SET_ZVAL_PTR op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX + } + | SET_ZVAL_PTR arg_addr, r0 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX + } + + |6: + | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, op_array, opline + |7: + + return 1; +} + +static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info; + uint32_t arg_num = opline->op2.num; + zend_jit_addr op1_addr, arg_addr; + + if ((opline->opcode == ZEND_SEND_VAR_EX || + opline->opcode == ZEND_SEND_VAR_NO_REF_EX) && + arg_num > MAX_ARG_FLAG_NUM) { + goto fallback; + } + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + op1_info = OP1_INFO(); + + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | mov RX, EX->call + } + + if (opline->opcode == ZEND_SEND_VAR_EX || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + + | mov r0, EX:RX->func + if (arg_num <= MAX_ARG_FLAG_NUM) { + | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask + | jnz >1 + } else { + ZEND_ASSERT(0); + } + + |.cold_code + |1: + + if (opline->opcode == ZEND_SEND_VAR_EX) { + if (!zend_jit_send_ref(Dst, opline, op_array, ssa, 1)) { + return 0; + } + } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { + mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); + + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 + if (op1_info & MAY_BE_REF) { + | cmp cl, IS_REFERENCE + | je >7 + } + | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask + | jnz >7 + | SAVE_VALID_OPLINE opline + | LOAD_ZVAL_ADDR FCARG1a, arg_addr + | EXT_CALL zend_jit_only_vars_by_reference, r0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } else { + ZEND_ASSERT(0); + } + + | jmp >7 + |.code + } + + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 + |.cold_code + |1: + } + + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | jmp >7 + |.code + } + } + + if (opline->opcode == ZEND_SEND_VAR_NO_REF) { + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R1, ZREG_R2 + if (op1_info & MAY_BE_REF) { + | cmp cl, IS_REFERENCE + | je >7 + } + | SAVE_VALID_OPLINE opline + | LOAD_ZVAL_ADDR FCARG1a, arg_addr + | EXT_CALL zend_jit_only_vars_by_reference, r0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } else if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + if (op1_info & MAY_BE_REF) { + if (opline->op1_type == IS_CV) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | ZVAL_DEREF FCARG1a, op1_info + | ZVAL_COPY_VALUE arg_addr, -1, val_addr, op1_info, ZREG_R0, ZREG_R2 + | TRY_ADDREF op1_info, ah, r2 + } else { + zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8); + + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 + |.cold_code + |1: + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR FCARG1a, op1_addr + | // ZVAL_COPY_VALUE(return_value, &ref->value); + | ZVAL_COPY_VALUE arg_addr, -1, ref_addr, op1_info, ZREG_R0, ZREG_R2 + | GC_DELREF FCARG1a + | je >1 + | IF_NOT_REFCOUNTED ah, >2 + | GC_ADDREF r2 + | jmp >2 + |1: + | EFREE_REG_24 op_array, opline + | jmp >2 + |.code + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + |2: + } + } else { + if (ra + && ssa->ops[opline - op_array->opcodes].op1_def >= 0 + && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) { + zend_jit_addr op1_def_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ssa->ops[opline - op_array->opcodes].op1_def); + + if (!zend_jit_update_regs(Dst, op1_addr, op1_def_addr, op1_info, ra[ssa->ops[opline - op_array->opcodes].op1_use])) { + return 0; + } + if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { + op1_addr= op1_def_addr; + } + } + | ZVAL_COPY_VALUE arg_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + if (opline->op1_type == IS_CV) { + | TRY_ADDREF op1_info, ah, r2 + } + } + } + |7: + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, int jmp) +{ + uint32_t target_label; + + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + if (jmp) { + | jmp >7 + } + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + if (jmp) { + | jmp >7 + } + } + + return 1; +} + +static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int b, zend_op_array *op_array, zend_ssa *ssa, int jmp) +{ + uint32_t target_label; + + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + if (jmp) { + | jmp >7 + } + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + if (jmp) { + | jmp >7 + } + } + + return 1; +} + +static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + zend_bool smart_branch = 0; + uint32_t defined_label = (uint32_t)-1; + uint32_t undefined_label = (uint32_t)-1; + zval *zv = RT_CONSTANT(opline, opline->op1); + zend_jit_addr res_addr; + + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + smart_branch = 1; + } + + if (smart_branch) { + if ((opline+1)->opcode == ZEND_JMPZ) { + undefined_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + defined_label = ssa->cfg.blocks[b].successors[0]; + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + undefined_label = ssa->cfg.blocks[b].successors[0]; + defined_label = ssa->cfg.blocks[b].successors[1]; + } else { + ZEND_ASSERT(0); + } + } + + | // if (CACHED_PTR(opline->extended_value)) { + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + opline->extended_value] + | test r0, r0 + | jz >1 + | test r0, 0x1 + | jnz >4 + |.cold_code + |4: + | MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, zend_constants, FCARG1a + | shl r0, 1 + | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax + if (smart_branch) { + if (undefined_label != (uint32_t)-1) { + | jz =>undefined_label + } else { + | jz >3 + } + } else { + | jz >2 + } + |1: + | SAVE_VALID_OPLINE opline + | LOAD_ADDR FCARG1a, zv + | EXT_CALL zend_jit_check_constant, r0 + | test r0, r0 + if (smart_branch) { + if (undefined_label != (uint32_t)-1) { + | jnz =>undefined_label + } else { + | jnz >3 + } + if (defined_label != (uint32_t)-1) { + | jmp =>defined_label + } else { + | jmp >3 + } + } else { + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + | jz >1 + |2: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + | jmp >3 + } + |.code + if (smart_branch) { + if (defined_label != (uint32_t)-1) { + | jmp =>defined_label + } + } else { + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + } + |3: + + return 1; +} + +static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, mask; + uint32_t target_label; + zend_uchar type; + zend_bool smart_branch = 0; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + + if (opline->extended_value & MAY_BE_RESOURCE) { + // TODO: support for is_resource() ??? + goto fallback; + } + + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + smart_branch = 1; + } + + op1_info = OP1_INFO(); + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 + |.cold_code + |1: + } + | SAVE_VALID_OPLINE opline + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + if (opline->extended_value & MAY_BE_NULL) { + if (!zend_jit_smart_true(Dst, opline, b, op_array, ssa, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0)) { + return 0; + } + } else { + if (!zend_jit_smart_false(Dst, opline, b, op_array, ssa, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0)) { + return 0; + } + } + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + |.code + } + } + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + mask = opline->extended_value; + switch (mask) { + case MAY_BE_NULL: type = IS_NULL; break; + case MAY_BE_FALSE: type = IS_FALSE; break; + case MAY_BE_TRUE: type = IS_TRUE; break; + case MAY_BE_LONG: type = IS_LONG; break; + case MAY_BE_DOUBLE: type = IS_DOUBLE; break; + case MAY_BE_STRING: type = IS_STRING; break; + case MAY_BE_ARRAY: type = IS_ARRAY; break; + case MAY_BE_OBJECT: type = IS_OBJECT; break; + default: + type = 0; + } + + if (!(op1_info & (MAY_BE_ANY - mask))) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + if (!zend_jit_smart_true(Dst, opline, b, op_array, ssa, 0)) { + return 0; + } + } else if (!(op1_info & mask)) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + if (!zend_jit_smart_false(Dst, opline, b, op_array, ssa, 0)) { + return 0; + } + } else { + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR r0, op1_addr + | ZVAL_DEREF r0, op1_info + } + if (type == 0) { + if (smart_branch && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (Z_REFCOUNTED_P(cv)) { + | IF_ZVAL_REFCOUNTED op1_addr, >1 + |.cold_code + |1: + } + | // if (!Z_DELREF_P(cv)) { + | GET_ZVAL_PTR FCARG1a, op1_addr + | GC_DELREF FCARG1a + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + | jnz >3 + } + if (op1_info & MAY_BE_REF) { + | mov al, byte [r0 + 8] + } else { + | mov al, byte [FP + opline->op1.var + 8] + } + | mov byte T1, al // save + | // zval_dtor_func(r); + | ZVAL_DTOR_FUNC op1_info, opline + | mov cl, byte T1 // restore + |jmp >2 + } + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (!RC_MAY_BE_1(op1_info)) { + | jmp >3 + } + |.code + } + |3: + if (op1_info & MAY_BE_REF) { + | mov cl, byte [r0 + 8] + } else { + | mov cl, byte [FP + opline->op1.var + 8] + } + |2: + } else { + if (op1_info & MAY_BE_REF) { + | mov cl, byte [r0 + 8] + } else { + | mov cl, byte [FP + opline->op1.var + 8] + } + } + | mov eax, 1 + | shl eax, cl + | test eax, mask + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | je =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jne =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | je =>target_label + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + | setne al + | movzx eax, al + | add eax, 2 + | SET_ZVAL_TYPE_INFO res_addr, eax + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + } + } else { + if (smart_branch && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (Z_REFCOUNTED_P(cv)) { + | IF_ZVAL_REFCOUNTED op1_addr, >1 + |.cold_code + |1: + } + | // if (!Z_DELREF_P(cv)) { + | GET_ZVAL_PTR FCARG1a, op1_addr + | GC_DELREF FCARG1a + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + | jnz >3 + } + if (op1_info & MAY_BE_REF) { + | mov al, byte [r0 + 8] + } else { + | mov al, byte [FP + opline->op1.var + 8] + } + | mov byte T1, al // save + | // zval_dtor_func(r); + | ZVAL_DTOR_FUNC op1_info, opline + | mov cl, byte T1 // restore + |jmp >2 + } + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (!RC_MAY_BE_1(op1_info)) { + | jmp >3 + } + |.code + } + |3: + if (op1_info & MAY_BE_REF) { + | mov cl, byte [r0 + 8] + } else { + | mov cl, byte [FP + opline->op1.var + 8] + } + |2: + | cmp cl, type + } else { + if (op1_info & MAY_BE_REF) { + | cmp byte [r0 + 8], type + } else { + | cmp byte [FP + opline->op1.var + 8], type + } + } + if ((opline+1)->opcode == ZEND_JMPZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jne =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | je =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + target_label = ssa->cfg.blocks[b].successors[0]; + | jne =>target_label + target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else { + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + | sete al + | movzx eax, al + | add eax, 2 + | SET_ZVAL_TYPE_INFO res_addr, eax + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + } + } + } + } + + |7: + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_free_compiled_variables(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t i, j, info; + + // Use type inference to avoid useless zval_ptr_dtor() + for (i = 0 ; i < op_array->last_var; i++) { + if (ssa->vars && ssa->var_info) { + info = ssa->var_info[i].type; + for (j = op_array->last_var; j < ssa->vars_count; j++) { + if (ssa->vars[j].var == i) { + info |= ssa->var_info[j].type; + } + } + } else { + info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF; + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + /* Refcount may be increased by RETRUN opcode */ + if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { + for (j = 0; j < ssa->cfg.blocks_count; j++) { + if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && + ssa->cfg.blocks[j].len > 0) { + const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; + + if (opline->opcode == ZEND_RETURN) { + if (opline->op1_type == IS_CV && + opline->op1.var == (uint32_t)(uintptr_t)(ZEND_CALL_VAR_NUM(NULL, i))) { + info |= MAY_BE_RCN; + break; + } + } + } + } + } +#endif + + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + uint32_t offset = (uint32_t)(uintptr_t)ZEND_CALL_VAR_NUM(NULL, i); + | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, 1, opline + } + } + return 1; +} + +static int zend_jit_leave_func(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + // Avoid multiple leave sequnces + if (jit_return_label >= 0) { + | jmp =>jit_return_label + return 1; + } + + jit_return_label = ssa->cfg.blocks_count * 2; + + |=>jit_return_label: + + // i_free_compiled_variables(execute_data); + if (!zend_jit_free_compiled_variables(Dst, opline, op_array, ssa)) { + return 0; + } + + /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ + | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] + | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_FAKE_CLOSURE) + | jnz ->leave_function_handler + + if ((op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) || + (op_array->fn_flags & ZEND_ACC_CLOSURE)) { + | // EG(current_execute_data) = EX(prev_execute_data); + | mov r0, EX->prev_execute_data + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2 + if (op_array->fn_flags & ZEND_ACC_CLOSURE) { + | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); + | mov r0, EX->func + | sub r0, sizeof(zend_object) + | OBJ_RELEASE r0, ecx, >4 + } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) { + | // if (call_info & ZEND_CALL_RELEASE_THIS) + | test FCARG1d, ZEND_CALL_RELEASE_THIS + | je >4 + | // zend_object *object = Z_OBJ(execute_data->This); + | mov r0, EX->This.value.obj + | // OBJ_RELEASE(object); + | OBJ_RELEASE r0, ecx, >4 + } + |4: + | // EG(vm_stack_top) = (zval*)execute_data; + | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0 + | // execute_data = EX(prev_execute_data); + | mov FP, EX->prev_execute_data + } else { + | // EG(vm_stack_top) = (zval*)execute_data; + | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0 + | // execute_data = EX(prev_execute_data); + | mov FP, EX->prev_execute_data + | // EG(current_execute_data) = execute_data + | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 + } + | // if (EG(exception)) + | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 + | LOAD_OPLINE + | jne ->leave_throw_handler + | // opline = EX(opline) + 1 + | ADD_IP sizeof(zend_op) + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | add r4, HYBRID_SPAD +#ifdef CONTEXT_THREADED_JIT + | push aword [IP] + | ret +#else + | JMP_IP +#endif + } else if (GCC_GLOBAL_REGS) { + | add r4, SPAD // stack alignment +#ifdef CONTEXT_THREADED_JIT + | push aword [IP] + | ret +#else + | JMP_IP +#endif + } else { +#ifdef CONTEXT_THREADED_JIT + ZEND_ASSERT(0); + // TODO: context threading can't work without GLOBAL REGS because we have to change + // the value of execute_data in execute_ex() + | mov FCARG1a, FP + | mov r0, aword [FP] + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | push aword [r0] + | ret +#else + | mov FP, aword T2 // restore FP + | mov RX, aword T3 // restore IP + | add r4, NR_SPAD // stack alignment + | mov r0, 2 // ZEND_VM_LEAVE + | ret +#endif + } + + return 1; +} + +static int zend_jit_return(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_lifetime_interval **ra) +{ + uint32_t op1_info; + zend_jit_addr op1_addr, ret_addr; + + if (op_array->type == ZEND_EVAL_CODE || !op_array->function_name || !ssa->ops || !ssa->var_info) { + // TODO: support for top-level code + return zend_jit_tail_handler(Dst, opline); + } + + op1_info = OP1_INFO(); + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, ra, ra ? ssa->ops[opline - op_array->opcodes].op1_use : -1); + if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { + // TODO: support for IS_UNDEF ??? + return zend_jit_tail_handler(Dst, opline); + } + + // if (!EX(return_value)) + if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { + | mov r2, EX->return_value + | test r2, r2 + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); + } else { + | mov r1, EX->return_value + | test r1, r1 + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); + } + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | jz >1 + |.cold_code + |1: + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (jit_return_label >= 0) { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label + } else { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 + } + } + | GET_ZVAL_PTR FCARG1a, op1_addr + | GC_DELREF FCARG1a + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + if (jit_return_label >= 0) { + | jnz =>jit_return_label + } else { + | jnz >9 + } + } + | //SAVE_OPLINE() + | ZVAL_DTOR_FUNC op1_info, opline + | //????mov r1, EX->return_value // reload ??? + } + if (jit_return_label >= 0) { + | jmp =>jit_return_label + } else { + | jmp >9 + } + |.code + } else { + if (jit_return_label >= 0) { + | jz =>jit_return_label + } else { + | jz >9 + } + } + + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + | ZVAL_COPY_CONST ret_addr, -1, zv, r0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r0 + } + } else if (opline->op1_type == IS_TMP_VAR) { + | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + } else if (opline->op1_type == IS_CV) { + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR r0, op1_addr + | ZVAL_DEREF r0, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); + } + | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + | // TODO: JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); ??? + | TRY_ADDREF op1_info, ah, r2 + } else { + if (op1_info & MAY_BE_REF) { + zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); + + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 + |.cold_code + |1: + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR r0, op1_addr + | // ZVAL_COPY_VALUE(return_value, &ref->value); + | ZVAL_COPY_VALUE ret_addr, -1, ref_addr, op1_info, ZREG_R2, ZREG_R2 + | GC_DELREF r0 + | je >2 + | // if (IS_REFCOUNTED()) + if (jit_return_label >= 0) { + | IF_NOT_REFCOUNTED dh, =>jit_return_label + } else { + | IF_NOT_REFCOUNTED dh, >9 + } + | // ADDREF + | GET_ZVAL_PTR r2, ret_addr // reload + | GC_ADDREF r2 + if (jit_return_label >= 0) { + | jmp =>jit_return_label + } else { + | jmp >9 + } + |2: + | EFREE_24 r0, op_array, opline + if (jit_return_label >= 0) { + | jmp =>jit_return_label + } else { + | jmp >9 + } + |.code + } + | ZVAL_COPY_VALUE ret_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_R2 + } + + |9: + //JIT: ZEND_VM_DISPATCH_TO_HELPER(zend_leave_helper); + return zend_jit_leave_func(Dst, opline, op_array, ssa); +} + +static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info, res_info; + zend_jit_addr op1_addr, op2_addr, res_addr; + + if (!ssa->ops || !ssa->var_info) { + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + res_info = RES_INFO(); + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 + } + | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, (opline->opcode == ZEND_FETCH_DIM_R) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, 8, 9)) { + return 0; + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + + if (op1_info & MAY_BE_STRING) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 + } + | SAVE_VALID_OPLINE opline + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + |.if X64 + | LOAD_ZVAL_ADDR CARG3, res_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR res_addr, r0 + |.endif + if (opline->opcode == ZEND_FETCH_DIM_R) { + | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 + } else if (opline->opcode == ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 + } else { + ZEND_ASSERT(0); + } + |.if not(X64) + | add r4, 12 + |.endif + if ((op1_info & MAY_BE_ARRAY) || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { + | jmp >9 // END + } + |6: + } + + if (op1_info & MAY_BE_OBJECT) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 + } + | SAVE_VALID_OPLINE opline + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } + |.if X64 + | LOAD_ZVAL_ADDR CARG3, res_addr + |.else + | sub r4, 12 + | PUSH_ZVAL_ADDR res_addr, r0 + |.endif + if (opline->opcode == ZEND_FETCH_DIM_R) { + | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 + } else if (opline->opcode == ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 + } else { + ZEND_ASSERT(0); + } + |.if not(X64) + | add r4, 12 + |.endif + if ((op1_info & MAY_BE_ARRAY) || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)))) { + | jmp >9 // END + } + |6: + } + + if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { + | SAVE_VALID_OPLINE opline + if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 + | // zend_error(E_NOTICE, "Undefined variable: %s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + + if (op2_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 + | mov FCARG1d, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT))) { + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL + if (op1_info & MAY_BE_ARRAY) { + | jmp >9 // END + } + } + + if (op1_info & MAY_BE_ARRAY) { + |.code + } + } + + if (op1_info & MAY_BE_ARRAY) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); + + |8: + if (op1_info & MAY_BE_ARRAY_OF_REF) { + | // ZVAL_COPY_DEREF + | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 + | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 + | GET_Z_PTR r0, r0 + | add r0, offsetof(zend_reference, val) + | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 + |1: + | GET_Z_PTR r1, r0 + | GC_ADDREF r1 + |2: + | ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 + } else { + | // ZVAL_COPY + | ZVAL_COPY_VALUE res_addr, -1, val_addr, MAY_BE_ANY, ZREG_R1, ZREG_R2 + | TRY_ADDREF res_info, ch, r2 + } + } + |9: // END + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { + /* Magic offsetGet() may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline + + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, int b, int *opnum, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info, op2_info; + zend_jit_addr op1_addr, op2_addr, res_addr; + + if (!ssa->ops || !ssa->var_info || (opline->extended_value & ZEND_ISEMPTY)) { + // TODO: support for empty() ??? + goto fallback; + } + + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + + op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + op2_addr = zend_jit_decode_op(op_array, opline->op2_type, opline->op2, opline, NULL, -1); + res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | ZVAL_DEREF FCARG1a, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 + } + | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, op_array, BP_JIT_IS, op1_info, op2_info, 8, 9)) { + return 0; + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + + | SAVE_VALID_OPLINE opline + if (Z_REG(op1_addr) != ZREG_FCARG1a) { + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + } + if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2a, op2_addr + } + | EXT_CALL zend_jit_isset_dim_helper, r0 + | test r0, r0 + | jz >9 + | jmp >8 + + if (op1_info & MAY_BE_ARRAY) { + |.code + } + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { + /* Magic offsetExists() may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + |8: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (!(opline->extended_value & ZEND_ISEMPTY)) { + if ((opline+1)->opcode == ZEND_JMPZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[1]; + | jmp =>target_label + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE + | jmp >8 + } + } else { + | //???? + | int3 + } + + |9: // not found + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, op_array, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, op_array, opline + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (!(opline->extended_value & ZEND_ISEMPTY)) { + if ((opline+1)->opcode == ZEND_JMPZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else if ((opline+1)->opcode == ZEND_JMPNZ) { + } else if ((opline+1)->opcode == ZEND_JMPZNZ) { + unsigned int target_label = ssa->cfg.blocks[b].successors[0]; + | jmp =>target_label + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE + } + } else { + | //???? + | int3 + } + + |8: + + if (((opline+1)->opcode == ZEND_JMPZ || + (opline+1)->opcode == ZEND_JMPNZ || + (opline+1)->opcode == ZEND_JMPZNZ) && + (opline+1)->op1_type == IS_TMP_VAR && + (opline+1)->op1.var == opline->result.var) { + (*opnum)++; + } + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zval *varname = RT_CONSTANT(opline, opline->op2); + + if (!ssa->ops || !ssa->var_info) { + op1_info = MAY_BE_ANY|MAY_BE_REF; + } else { + op1_info = OP1_INFO(); + } + + //idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; + | mov r0, EX->run_time_cache + | mov r0, aword [r0 + opline->extended_value] + | sub r0, 1 + //if (EXPECTED(idx < EG(symbol_table).nNumUsed)) + | MEM_OP2_2_ZTS cmp, eax, dword, executor_globals, symbol_table.nNumUsed, r1 + | jae >9 + //Bucket *p = EG(symbol_table).arData + idx; + |.if X64 + | shl r0, 5 + |.else + | imul r0, sizeof(Bucket) + |.endif + | MEM_OP2_2_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 + | IF_Z_TYPE r0, IS_UNDEF, >9 + // (EXPECTED(p->key == Z_STR_P(varname)) + | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, key)], Z_PTR_P(varname), r1 + | jne >1 + |.cold_code + |1: + //(EXPECTED(p->h == ZSTR_H(Z_STR_P(varname))) + | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, h)], ZSTR_H(Z_STR_P(varname)), r1 + | jne >9 + //EXPECTED(p->key != NULL) + | mov r1, [r0 + offsetof(Bucket, key)] + | test r1, r1 + | jz >9 + //EXPECTED(ZSTR_LEN(p->key) == Z_STRLEN_P(varname)) + | ADDR_OP2_2 cmp, aword [r1 + offsetof(zend_string, len)], Z_STRLEN_P(varname), r2 + | jne >9 + //EXPECTED(memcmp(ZSTR_VAL(p->key), Z_STRVAL_P(varname), Z_STRLEN_P(varname)) == 0) + | add r1, offsetof(zend_string, val) + | mov T1, r0 + |.if X64 + | mov CARG1, r1 + | LOAD_ADDR CARG2, Z_STRVAL_P(varname) + | mov CARG3, Z_STRLEN_P(varname) + | EXT_CALL memcmp, r0 + |.else + | sub r4, 4 + | push Z_STRLEN_P(varname) + | push Z_STRVAL_P(varname) + | push r1 + | call &memcmp + | add r4, 16 + |.endif + | test al, al + | mov r0, aword T1 + | jnz >9 + | jmp >2 + |.code + |2: + // if (UNEXPECTED(Z_TYPE_P(value) == IS_INDIRECT)) + | mov cl, byte [r0 + 8] + | cmp cl, IS_INDIRECT + | je >1 + |.cold_code + |1: + //value = Z_INDIRECT_P(value) + | mov r0, [r0] + | mov cl, byte [r0 + 8] + | test cl, cl // cmp cl, IS_UNDEF + | jne >2 + | SET_Z_TYPE_INFO r0, IS_NULL + | jmp >8 + |.code + |2: + | cmp cl, IS_REFERENCE + | jne >8 + |1: + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + //stash this for later use + | mov r2, r0 + } + | GET_Z_PTR r0, r0 + | GC_ADDREF r0 + //if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (op1_info & (MAY_BE_ANY - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_ZVAL_REFCOUNTED op1_addr, >2 + } + |.cold_code + //zval_dtor_func(Z_COUNTED_P(variable_ptr)) + |2: + //if (EXPECTED(variable_ptr != value)) + | LOAD_ZVAL_ADDR FCARG1a, op1_addr + | cmp FCARG1a, r2 + | je >4 + | GET_Z_PTR FCARG1a, FCARG1a + | GC_DELREF FCARG1a + if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { + | jnz >3 + } + | mov aword T1, r0 // save + | ZVAL_DTOR_FUNC op1_info, opline + | mov r0, aword T1 // restore + | jmp >5 + if (op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { + |3: + // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) + | IF_GC_MAY_NOT_LEAK FCARG1a, edx, >5 + | mov aword T1, r0 //save + | EXT_CALL gc_possible_root, r1 + | mov r0, aword T1 // restore + | jmp >5 + } + |4: + | GET_Z_PTR FCARG1a, FCARG1a + | GC_DELREF FCARG1a + | jmp >5 + |.code + } + |5: + //ZVAL_REF(variable_ptr, ref) + | SET_ZVAL_PTR op1_addr, r0 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX + //END of handler + + |.cold_code + |8: + | mov FCARG1a, r0 + | EXT_CALL zend_jit_new_ref_helper, r0 + | jmp <1 + |9: + | mov FCARG1a, FP + | LOAD_ADDR FCARG2a, (ptrdiff_t)varname + |.if X64 + | mov CARG3, opline->extended_value + |.else + | sub r4, 12 + | push opline->extended_value + |.endif + | EXT_CALL zend_jit_fetch_global_helper, r0 + |.if not(X64) + | add r4, 12 + |.endif + | jmp <1 + |.code + + return 1; +} + +static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t arg_num = opline->op1.num; + + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + zend_arg_info *arg_info = NULL; + + if (EXPECTED(arg_num <= op_array->num_args)) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { + arg_info = &op_array->arg_info[op_array->num_args]; + } + if (arg_info) { + zend_type type = arg_info->type; + + if (ZEND_TYPE_IS_SET(type)) { + // Type check + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + | LOAD_ZVAL_ADDR r0, res_addr + if (arg_info->pass_by_reference) { + | GET_Z_PTR r0, r0 + | add r0, offsetof(zend_reference, val) + } + if (!ZEND_TYPE_IS_CLASS(type)) { + unsigned char code = ZEND_TYPE_CODE(type); + + if (code == _IS_BOOL) { + | cmp byte [r0 + 8], IS_FALSE + | je >1 + | cmp byte [r0 + 8], IS_TRUE + | jne >8 + |1: + } else { + | cmp byte [r0 + 8], code + | jne >8 + } + } else { + | SAVE_VALID_OPLINE opline + | cmp byte [r0 + 8], IS_OBJECT + | jne >9 + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | add r0, opline->op2.num + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64WIN + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov aword A5, r0 + | EXT_CALL zend_jit_verify_arg_object, r0 + |.elif X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + | EXT_CALL zend_jit_verify_arg_object, r0 + |.else + | sub r4, 4 + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num + | EXT_CALL zend_jit_verify_arg_object, r0 + | add r4, 4 + |.endif + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + |.cold_code + |8: + | SAVE_VALID_OPLINE opline + |9: + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | add r0, opline->op2.num + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64WIN + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov aword A5, r0 + | mov aword A6, 0 + | EXT_CALL zend_jit_verify_arg_slow, r0 + |.elif X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + | xor CARG6, CARG6 + | EXT_CALL zend_jit_verify_arg_slow, r0 + |.else + | push 0 + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num + | EXT_CALL zend_jit_verify_arg_slow, r0 + |.endif + if (!zend_jit_check_exception(Dst)) { + return 0; + } + | jmp >1 + |.code + |1: + + if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { + last_valid_opline = NULL; + if (!zend_jit_set_valid_ip(Dst, opline + 1)) { + return 0; + } + } + + return 1; + } + } + } + + | cmp dword EX->This.u2.num_args, arg_num + | jb >1 + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | mov FCARG1a, FP + | EXT_CALL zend_missing_arg_error, r0 + | jmp ->exception_handler + |.code + + if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { + last_valid_opline = NULL; + if (!zend_jit_set_valid_ip(Dst, opline + 1)) { + return 0; + } + } + + return 1; +} + +static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_bool is_last, zend_ssa *ssa) +{ + zend_arg_info *arg_info = NULL; + zend_bool has_slow = 0; + uint32_t arg_num = opline->op1.num; + zval *zv = RT_CONSTANT(opline, opline->op2); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + + | cmp dword EX->This.u2.num_args, arg_num + | jae >5 + | ZVAL_COPY_CONST res_addr, -1, zv, r0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, r0 + } + if (Z_CONSTANT_P(zv)) { + has_slow = 1; + | SAVE_VALID_OPLINE opline + |.if X64 + | LOAD_ZVAL_ADDR CARG1, res_addr + | mov r0, EX->func + | mov CARG2, [r0 + offsetof(zend_op_array, scope)] + | EXT_CALL zval_update_constant_ex, r0 + |.else + | sub r4, 8 + | mov r0, EX->func + | push dword [r0 + offsetof(zend_op_array, scope)] + | LOAD_ZVAL_ADDR r0, res_addr + | push r0 + | EXT_CALL zval_update_constant_ex, r0 + | add r4, 16 + |.endif + | test al, al + | jnz >7 + } + |5: + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + do { + if (arg_num <= op_array->num_args) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { + arg_info = &op_array->arg_info[op_array->num_args]; + } else { + break; + } + if (!ZEND_TYPE_IS_SET(arg_info->type)) { + break; + } + has_slow += 2; + | LOAD_ZVAL_ADDR r0, res_addr + | ZVAL_DEREF r0, MAY_BE_REF + if (!ZEND_TYPE_IS_CLASS(arg_info->type)) { + | cmp byte [r0 + 8], ZEND_TYPE_CODE(arg_info->type) + | jne >8 + } else { + | cmp byte [r0 + 8], IS_OBJECT + | jne >8 + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | lea r0, [r0 + opline->extended_value] + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64WIN + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov aword A5, r0 + | EXT_CALL zend_jit_verify_arg_object, r0 + |.elif X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + | EXT_CALL zend_jit_verify_arg_object, r0 + |.else + | sub r4, 4 + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num + | EXT_CALL zend_jit_verify_arg_object, r0 + | add r4, 4 + |.endif + } + } while (0); + } + |9: + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + if (is_last) { + | LOAD_IP_ADDR (opline + 1) + last_valid_opline = (opline + 1); + } + + if (has_slow) { + |.cold_code + if (has_slow & 1) { + |7: + | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, 0, opline + | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + | jmp <5 + } + if (has_slow & 2) { + |8: + | mov FCARG1a, r0 + | mov r0, EX->run_time_cache + | lea r0, [r0 + opline->extended_value] + | LOAD_ADDR FCARG2a, (ptrdiff_t)op_array + |.if X64WIN + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov aword A5, r0 + | ADDR_OP2_2 mov, aword A6, zv, r0 + | EXT_CALL zend_jit_verify_arg_slow, r0 + |.elif X64 + | mov CARG3, arg_num + | LOAD_ADDR CARG4, (ptrdiff_t)arg_info + | mov CARG5, r0 + | LOAD_ADDR CARG6, zv + | EXT_CALL zend_jit_verify_arg_slow, r0 + |.else + | push zv + | push r0 + | push (ptrdiff_t)arg_info + | push arg_num + | EXT_CALL zend_jit_verify_arg_slow, r0 + |.endif + | jmp <9 + } + |.code + } + + return 1; +} + +#define ZEND_WRONG_PROPERTY_OFFSET 0 + +static uint32_t zend_get_known_property_offset(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) +{ + zend_property_info *info; + + if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || (ce->ce_flags & ZEND_ACC_TRAIT)) { + return ZEND_WRONG_PROPERTY_OFFSET; + } + + if (ce->info.user.filename != filename) { + /* class declaration might be changed infdependently */ + return ZEND_WRONG_PROPERTY_OFFSET; + } + + if (ce->ce_flags & ZEND_ACC_INHERITED) { + if (!ce->parent) { + /* propery offests may be changed by inheritance */ + return ZEND_WRONG_PROPERTY_OFFSET; + } else { + zend_class_entry *parent = ce->parent; + + do { + if (parent->type == ZEND_INTERNAL_CLASS) { + break; + } else if (parent->info.user.filename != filename) { + /* some of parents class declarations might be changed infdependently */ + /* TODO: this check may be not enough, because even + * in the same it's possible to conditionnaly define + * few classes with the same name, and "parent" may + * change from request to request. + */ + return ZEND_WRONG_PROPERTY_OFFSET; + } + parent = parent->parent; + } while (parent); + } + } + + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + if (info == NULL || + info->offset == ZEND_WRONG_PROPERTY_OFFSET || + (info->flags & ZEND_ACC_STATIC)) { + return ZEND_WRONG_PROPERTY_OFFSET; + } + + if (!(info->flags & ZEND_ACC_PUBLIC) && + (!on_this || info->ce != ce)) { + return ZEND_WRONG_PROPERTY_OFFSET; + } + + return info->offset; +} + +static zend_bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) +{ + zend_property_info *info; + + if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { + return 1; + } + + if (ce->info.user.filename != filename) { + /* class declaration might be changed infdependently */ + return 1; + } + + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + if (info == NULL || + info->offset == ZEND_WRONG_PROPERTY_OFFSET || + (info->flags & ZEND_ACC_STATIC)) { + return 1; + } + + if (!(info->flags & ZEND_ACC_PUBLIC) && + (!on_this || info->ce != ce)) { + return 1; + } + + return 0; +} + +static int zend_jit_fetch_obj_read(dasm_State **Dst, zend_op *opline, zend_op_array *op_array, zend_ssa *ssa, zend_bitset checked_this) +{ + uint32_t op1_info; + zend_class_entry *ce = NULL; + zval *member; + uint32_t offset; + zend_bool may_be_dynamic = 1; + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + zend_jit_addr res_addr = zend_jit_decode_op(op_array, opline->result_type, opline->result, opline, NULL, -1); + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr prop_addr; + + if (opline->op2_type != IS_CONST) { + goto fallback; + } + + member = RT_CONSTANT(opline, opline->op2); + if (Z_TYPE_P(member) != IS_STRING || Z_STRVAL_P(member)[0] == '\0') { + goto fallback; + } + + if (opline->op1_type == IS_UNUSED) { + op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; + ce = op_array->scope; + } else { + op1_info = OP1_INFO(); + if (ssa->var_info && ssa->ops) { + zend_ssa_var_info *op1_ssa = + ssa->var_info + ssa->ops[opline - op_array->opcodes].op1_use; + + if (op1_ssa->ce && !op1_ssa->is_instanceof && !op1_ssa->ce->create_object) { + ce = op1_ssa->ce; + } + } + } + + if (!(op1_info & MAY_BE_OBJECT)) { + goto fallback; + } + + offset = zend_get_known_property_offset(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED) { + if (!checked_this || !zend_bitset_in(checked_this, (opline - op_array->opcodes))) { + | IF_ZVAL_TYPE this_addr, IS_UNDEF, >1 + |.cold_code + |1: + | SAVE_VALID_OPLINE opline + | jmp ->not_obj + |.code + } + | GET_ZVAL_PTR FCARG1a, this_addr + } else { + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR r0, op1_addr + | ZVAL_DEREF r0, op1_info + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 + } + | GET_ZVAL_PTR FCARG1a, op1_addr + } + + if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + | mov r0, EX->run_time_cache + | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] + | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] + | jne >5 + | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] + may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + if (may_be_dynamic) { + | test r0, r0 + | jl >8 // dynamic property + } + | mov edx, dword [FCARG1a + r0 + 8] + | IF_TYPE dl, IS_UNDEF, >5 + | add FCARG1a, r0 + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offset); + | mov edx, dword [FCARG1a + offset + 8] + | IF_TYPE dl, IS_UNDEF, >5 + } + | GET_ZVAL_PTR r0, prop_addr + | IF_NOT_REFCOUNTED dh, >2 + if (opline->opcode == ZEND_FETCH_OBJ_R || opline->opcode == ZEND_FETCH_OBJ_IS) { + | IF_TYPE dl, IS_REFERENCE, >6 + } + |1: + | GC_ADDREF r0 + |2: + |.if X64 + | SET_ZVAL_PTR res_addr, r0 + |.else + | SET_ZVAL_PTR res_addr, r0 + | GET_ZVAL_W2 r0, prop_addr + | SET_ZVAL_W2 res_addr, r0 + |.endif + | SET_ZVAL_TYPE_INFO res_addr, edx + + |.cold_code + |5: + | LOAD_ADDR FCARG2a, member + |.if X64 + | LOAD_ZVAL_ADDR CARG3, res_addr + | mov CARG4, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + |.else + | sub r4, 8 + | push (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + | PUSH_ZVAL_ADDR res_addr, r0 + |.endif + | SAVE_VALID_OPLINE opline + if (opline->opcode == ZEND_FETCH_OBJ_R) { + | EXT_CALL zend_jit_fetch_obj_r_slow, r0 + } else if (opline->opcode == ZEND_FETCH_OBJ_IS) { + | EXT_CALL zend_jit_fetch_obj_is_slow, r0 + } else { + ZEND_ASSERT(0); + } + |.if not(X64) + | add r4, 8 + |.endif + | jmp >9 + + if (opline->opcode == ZEND_FETCH_OBJ_R || opline->opcode == ZEND_FETCH_OBJ_IS) { + |6: + if (offset == ZEND_WRONG_PROPERTY_OFFSET) { + | mov FCARG2a, FCARG1a + } else { + | lea FCARG2a, [FCARG1a + offset] + } + | LOAD_ZVAL_ADDR FCARG1a, res_addr + | EXT_CALL zend_jit_zval_copy_deref_helper, r0 + | jmp >9 + } + + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) { + |7: + if (opline->opcode == ZEND_FETCH_OBJ_R) { + | SAVE_VALID_OPLINE opline + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & MAY_BE_ANY) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 + } + | mov FCARG1d, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, r0 + |1: + } + |.if X64 + | mov CARG1, E_NOTICE + | LOAD_ADDR CARG2, "Trying to get property '%s' of non-object" + | LOAD_ADDR CARG3, Z_STRVAL_P(member) + | EXT_CALL zend_error, r0 + |.else + | sub r4, 4 + | push Z_STRVAL_P(member) + | push "Trying to get property '%s' of non-object" + | push E_NOTICE + | EXT_CALL zend_error, r0 + | add r4, 16 + |.endif + } + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL + | jmp >9 + } + + if (offset == ZEND_WRONG_PROPERTY_OFFSET && may_be_dynamic) { + |8: + | mov FCARG2a, r0 + |.if X64WIN + | LOAD_ADDR CARG3, member + | LOAD_ZVAL_ADDR CARG4, res_addr + | mov aword A5, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + |.elif X64 + | LOAD_ADDR CARG3, member + | LOAD_ZVAL_ADDR CARG4, res_addr + | mov CARG5, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + |.else + | sub r4, 4 + | push (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + | PUSH_ZVAL_ADDR res_addr, r0 + | PUSH_ADDR member, r0 + |.endif + | SAVE_VALID_OPLINE opline + if (opline->opcode == ZEND_FETCH_OBJ_R) { + | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 + } else if (opline->opcode == ZEND_FETCH_OBJ_IS) { + | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 + } + |.if not(X64) + | add r4, 4 + |.endif + | jmp >9 + } + + |.code; + |9: // END + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, op_array, opline + + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + + return 1; + +fallback: + /* fallback to subroutine threading */ + return zend_jit_handler(Dst, opline, zend_may_throw(opline, op_array, ssa)); +} + +static int zend_jit_free(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + uint32_t op1_info = OP1_INFO(); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (zend_may_throw(opline, op_array, ssa)) { + | SAVE_VALID_OPLINE, opline + } + if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { + if (op1_info & MAY_BE_ARRAY) { + | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 + } + | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] + | cmp FCARG1d, -1 + | je >7 + | EXT_CALL zend_hash_iterator_del, r0 + |7: + } + | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, 0, opline + if (zend_may_throw(opline, op_array, ssa)) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + } + + return 1; +} + +static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + + if (Z_TYPE_P(zv) == IS_STRING) { + size_t len = Z_STRLEN_P(zv); + + if (len > 0) { + const char *str = Z_STRVAL_P(zv); + + | SAVE_VALID_OPLINE opline + |.if X64 + | LOAD_ADDR CARG1, str + | LOAD_ADDR CARG2, len + | EXT_CALL zend_write, r0 + |.else + | sub r4, 8 + | push len + | push str + | EXT_CALL zend_write, r0 + | add r4, 16 + |.endif + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; + } + } + + if (!zend_jit_handler(Dst, opline, 1)) { + return 0; + } + + return 1; +} + +static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, zend_op_array *op_array, zend_ssa *ssa) +{ + HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); + + if (sizeof(void*) == 8 && !IS_32BIT(dasm_end)) { + // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + return 1; + } + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + zval *jump_zv; + int b; + + if (opline->opcode == ZEND_SWITCH_LONG) { + if (Z_TYPE_P(zv) == IS_LONG) { + jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); + if (jump_zv != NULL) { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; + } else { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + } + | jmp =>b + } + } else if (opline->opcode == ZEND_SWITCH_STRING) { + if (Z_TYPE_P(zv) == IS_STRING) { + jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); + if (jump_zv != NULL) { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; + } else { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + } + | jmp =>b + } + } else { + ZEND_ASSERT(0); + } + } else { + uint32_t op1_info = OP1_INFO(); + zend_jit_addr op1_addr = zend_jit_decode_op(op_array, opline->op1_type, opline->op1, opline, NULL, -1); + int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + zval *val; + + if (opline->opcode == ZEND_SWITCH_LONG) { + if (op1_info & MAY_BE_LONG) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1 + | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr + |.cold_code + |1: + | // ZVAL_DEREF(op) + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 + | GET_ZVAL_PTR FCARG2a, op1_addr + | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 + | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] + | jmp >2 + |.code + |2: + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 + } + | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr + } + if (HT_IS_PACKED(jumptable)) { + uint32_t count = jumptable->nNumUsed; + Bucket *p = jumptable->arData; + + | cmp FCARG2a, jumptable->nNumUsed + | jae >3 + |.if X64 + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + | movsxd r0, dword [FCARG2a * 4 + >4] + | jmp r0 + |.else + | jmp aword [FCARG2a * 4 + >4] + |.endif + |3: + |.cold_code + |4: + p = jumptable->arData; + do { + if (Z_TYPE(p->val) == IS_UNDEF) { + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + | .aword =>b + } else { + int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)) - op_array->opcodes]; + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + | .aword =>b + } + p++; + count--; + } while (count); + |.code + } else { + | LOAD_ADDR FCARG1a, jumptable + | EXT_CALL zend_hash_index_find, r0 + | test r0, r0 + | jz =>b + | LOAD_ADDR FCARG1a, jumptable + | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + | mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t)) + |.if X64 + | cqo + |.else + | cdq + |.endif + | idiv FCARG1a + |.if X64 + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + | movsxd r0, dword [r0 + >4] + | jmp r0 + |.else + | jmp dword [r0 + >4] + |.endif + |3: + |.cold_code + |4: + ZEND_HASH_FOREACH_VAL(jumptable, val) { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes]; + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + | .aword =>b + } ZEND_HASH_FOREACH_END(); + |.code + } + } + } else if (opline->opcode == ZEND_SWITCH_STRING) { + if (op1_info & MAY_BE_STRING) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1 + | GET_ZVAL_PTR FCARG2a, op1_addr + |.cold_code + |1: + | // ZVAL_DEREF(op) + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 + | GET_ZVAL_PTR FCARG2a, op1_addr + | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 + | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] + | jmp >2 + |.code + |2: + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 + } + | GET_ZVAL_PTR FCARG2a, op1_addr + } + | LOAD_ADDR FCARG1a, jumptable + | EXT_CALL zend_hash_find, r0 + | test r0, r0 + | jz =>b + | LOAD_ADDR FCARG1a, jumptable + | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + | mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t)) + |.if X64 + | cqo + |.else + | cdq + |.endif + | idiv FCARG1a + |.if X64 + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + | movsxd r0, dword [r0 + >4] + | jmp r0 + |.else + | jmp dword [r0 + >4] + |.endif + |3: + |.cold_code + |4: + ZEND_HASH_FOREACH_VAL(jumptable, val) { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes]; + | // TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ??? + |.if X64 + | .aword =>b + |.else + | .aword =>b + |.endif + } ZEND_HASH_FOREACH_END(); + |.code + } + } else { + ZEND_ASSERT(0); + } + } + return 1; +} + +static zend_bool zend_jit_may_reuse_reg(zend_op_array *op_array, zend_ssa *ssa, uint32_t position, int def_var, int use_var) +{ + if (ssa->var_info[def_var].type != ssa->var_info[use_var].type) { + return 0; + } + + switch (op_array->opcodes[position].opcode) { + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: + case ZEND_ASSIGN: + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + return 1; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + if (def_var == ssa->ops[position].result_def && + use_var == ssa->ops[position].op1_use) { + return 1; + } + break; + default: + break; + } + return 0; +} + +static zend_bool zend_jit_opline_supports_reg(zend_op_array *op_array, zend_ssa *ssa, zend_op *opline, int var) +{ + uint32_t op1_info, op2_info; + + switch (opline->opcode) { + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_CASE: + case ZEND_RETURN: + return 1; + case ZEND_ASSIGN: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + return + opline->op1_type == IS_CV && + !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if ((op1_info | op2_info) & MAY_BE_UNDEF) { + return 0; + } + return (op1_info & op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) != 0; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if ((op1_info | op2_info) & MAY_BE_UNDEF) { + return 0; + } + return (op1_info & op2_info & MAY_BE_LONG) != 0; + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + op1_info = OP1_INFO(); + return + opline->op1_type == IS_CV && + (op1_info & MAY_BE_LONG); + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + return 1; + } + return 0; +} + +static zend_bool zend_jit_may_be_in_reg(zend_op_array *op_array, zend_ssa *ssa, int var) +{ + if (ssa->vars[var].no_val) { + /* we don't need the value */ + return 0; + } + + if (zend_jit_reg_alloc < ZEND_JIT_REG_ALLOC_GLOBAL) { + /* Disable global register allocation, + * register allocation forSAA variables connected through Phi functions + */ + if (ssa->vars[var].definition_phi) { + return 0; + } + if (ssa->vars[var].phi_use_chain) { + zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; + do { + if (!ssa->vars[phi->ssa_var].no_val) { + return 0; + } + phi = zend_ssa_next_use_phi(ssa, var, phi); + } while (phi); + } + } + + if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && + ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { + /* bad type */ + return 0; + } + + if (ssa->vars[var].definition >= 0) { + if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + ssa->vars[var].definition, var)) { + return 0; + } + } + + if (ssa->vars[var].use_chain >= 0) { + int use = ssa->vars[var].use_chain; + + do { + if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && + !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, var)) { + return 0; + } + use = zend_ssa_next_use(ssa->ops, var, use); + } while (use >= 0); + } + + return 1; +} + +static zend_bool zend_needs_extra_reg_for_const(zend_op_array *op_array, const zend_op *opline, zend_uchar op_type, znode_op op) +{ +|.if X64 +|| if (op_type == IS_CONST) { +|| zval *zv = RT_CONSTANT(opline, op); +|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_32BIT(zv)) { +|| return 1; +|| } +|| } +|.endif + return 0; +} + +static zend_regset zend_jit_get_scratch_regset(zend_op_array *op_array, zend_ssa *ssa, uint32_t line, int current_var) +{ + const zend_op *opline = op_array->opcodes + line; + uint32_t op1_info, op2_info, res_info; + zend_regset regset = ZEND_REGSET_SCRATCH; + + switch (opline->opcode) { + case ZEND_NOP: + case ZEND_OP_DATA: + case ZEND_JMP: + case ZEND_RETURN: + regset = ZEND_REGSET_EMPTY; + break; + case ZEND_QM_ASSIGN: + if (ssa->ops[line].op1_def == current_var || + ssa->ops[line].result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + /* break missing intentionally */ + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + if (ssa->ops[line].op1_use == current_var) { + regset = ZEND_REGSET(ZREG_R0); + break; + } + op1_info = OP1_INFO(); + if (!(op1_info & MAY_BE_UNDEF)) { + if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_XMM0); + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_R0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); + } + } + break; + case ZEND_SEND_VAR: + if (ssa->ops[line].op1_use == current_var || + ssa->ops[line].op1_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + if (!(op1_info & MAY_BE_UNDEF)) { + if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_XMM0); + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); + if (op1_info & MAY_BE_REF) { + ZEND_REGSET_INCL(regset, ZREG_R1); + } + } + } + break; + case ZEND_ASSIGN: + if (ssa->ops[line].op2_use == current_var || + ssa->ops[line].op2_def == current_var || + ssa->ops[line].op1_def == current_var || + ssa->ops[line].result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (opline->op1_type == IS_CV + && !(op2_info & MAY_BE_UNDEF) + && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF))) { + if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_XMM0); + } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_R0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); + } + } + break; + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + if (ssa->ops[line].op1_use == current_var || + ssa->ops[line].op1_def == current_var || + ssa->ops[line].result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + if (opline->op1_type == IS_CV + && (op1_info & MAY_BE_LONG) + && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + if (op1_info & MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_XMM0); + } + } + break; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + + regset = ZEND_REGSET_EMPTY; + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + res_info = OP1_INFO(); + if (res_info & MAY_BE_DOUBLE) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + ZEND_REGSET_INCL(regset, ZREG_XMM1); + } + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa->ops[line].result_def != current_var) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { + if (zend_is_commutative(opline->opcode)) { + if (ssa->ops[line].result_def != current_var) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + } else { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_XMM1); + } + } + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + } + if (zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op2)) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } + break; + case ZEND_SL: + case ZEND_SR: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + if (opline->op2_type != IS_CONST && ssa->ops[line].op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_R1); + } + } + break; + case ZEND_MOD: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + if (opline->op2_type == IS_CONST && + Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && + zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)))) { + if (ssa->ops[line].result_def != current_var && + (ssa->ops[line].op1_use != current_var || !zend_ssa_is_last_use(op_array, ssa, current_var, opline-op_array->opcodes))) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } else { + ZEND_REGSET_INCL(regset, ZREG_R0); + ZEND_REGSET_INCL(regset, ZREG_R1); + } + } + break; + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_CASE: + op1_info = OP1_INFO(); + op2_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && + opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { + if (ssa->ops[line].op1_use != current_var && + ssa->ops[line].op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa->ops[line].op1_use != current_var && + ssa->ops[line].op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + } + if (zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(op_array, opline, opline->op1_type, opline->op2)) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } + break; + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + op1_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + if (op1_info & MAY_BE_DOUBLE) { + ZEND_REGSET_INCL(regset, ZREG_XMM0); + } + if (opline->opcode == ZEND_BOOL || + opline->opcode == ZEND_BOOL_NOT || + opline->opcode == ZEND_JMPZ_EX || + opline->opcode == ZEND_JMPNZ_EX) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } + break; + case ZEND_DO_UCALL: + case ZEND_DO_FCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_INCLUDE_OR_EVAL: + case ZEND_GENERATOR_CREATE: + case ZEND_YIELD: + case ZEND_YIELD_FROM: + regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); + break; + default: + break; + } + +#if ZTS + /* %r0 is used to check EG(vm_interrupt) */ + { + uint32_t b = ssa->cfg.map[line]; + + if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 + && ssa->cfg.blocks[b].start == line) { + ZEND_REGSET_INCL(regset, ZREG_R0); + } + } +#endif + + return regset; +} + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ |