/* * +----------------------------------------------------------------------+ * | Zend JIT | * +----------------------------------------------------------------------+ * | Copyright (c) The PHP Group | * +----------------------------------------------------------------------+ * | This source file is subject to version 3.01 of the PHP license, | * | that is bundled with this package in the file LICENSE, and is | * | available through the world-wide-web at the following url: | * | http://www.php.net/license/3_01.txt | * | If you did not receive a copy of the PHP license and are unable to | * | obtain it through the world-wide-web, please send a note to | * | license@php.net so we can mail you a copy immediately. | * +----------------------------------------------------------------------+ * | Authors: Dmitry Stogov | * | Xinchen Hui | * +----------------------------------------------------------------------+ */ |.if X64 |.arch x64 |.else |.arch x86 |.endif |.if X64WIN |.define FP, r14 |.define IP, r15 |.define IPl, r15d |.define RX, r15 // the same as VM IP reused as a general purpose reg |.define CARG1, rcx // x64/POSIX C call arguments. |.define CARG2, rdx |.define CARG3, r8 |.define CARG4, r9 |.define CARG1d, ecx |.define CARG2d, edx |.define CARG3d, r8d |.define CARG4d, r9d |.define FCARG1a, CARG1 // Simulate x86 fastcall. |.define FCARG2a, CARG2 |.define FCARG1d, CARG1d |.define FCARG2d, CARG2d |.define SPAD, 0x58 // padding for CPU stack alignment |.define NR_SPAD, 0x58 // padding for CPU stack alignment |.define T3, [r4+0x50] // Used to store old value of IP |.define T2, [r4+0x48] // Used to store old value of FP |.define T1, [r4+0x40] |.define A6, [r4+0x28] // preallocated slot for 6-th argument |.define A5, [r4+0x20] // preallocated slot for 5-th argument |.elif X64 |.define FP, r14 |.define IP, r15 |.define IPl, r15d |.define RX, r15 // the same as VM IP reused as a general purpose reg |.define CARG1, rdi // x64/POSIX C call arguments. |.define CARG2, rsi |.define CARG3, rdx |.define CARG4, rcx |.define CARG5, r8 |.define CARG6, r9 |.define CARG1d, edi |.define CARG2d, esi |.define CARG3d, edx |.define CARG4d, ecx |.define CARG5d, r8d |.define CARG6d, r9d |.define FCARG1a, CARG1 // Simulate x86 fastcall. |.define FCARG2a, CARG2 |.define FCARG1d, CARG1d |.define FCARG2d, CARG2d |.define SPAD, 0x18 // padding for CPU stack alignment |.define NR_SPAD, 0x28 // padding for CPU stack alignment |.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only) |.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only) |.define T1, [r4] |.else |.define FP, esi |.define IP, edi |.define IPl, edi |.define RX, edi // the same as VM IP reused as a general purpose reg |.define FCARG1a, ecx // x86 fastcall arguments. |.define FCARG2a, edx |.define FCARG1d, ecx |.define FCARG2d, edx |.define SPAD, 0x1c // padding for CPU stack alignment |.define NR_SPAD, 0x1c // padding for CPU stack alignment |.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only) |.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only) |.define T1, [r4] |.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) |.define A3, [r4+0x8] |.define A2, [r4+0x4] |.define A1, [r4] |.endif |.define HYBRID_SPAD, 16 // padding for stack alignment #ifdef _WIN64 # define TMP_ZVAL_OFFSET 0x20 #else # define TMP_ZVAL_OFFSET 0 #endif #define DASM_ALIGNMENT 16 /* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to * guarantee proper alignment of 128-bit SSE data allocated on stack. * With broken alignment any execution of SSE code, including calls to * memcpy() and others, may lead to crash. */ #include "Zend/zend_cpuinfo.h" #ifdef HAVE_VALGRIND # include #endif /* The generated code may contain tautological comparisons, ignore them. */ #if defined(__clang__) # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wtautological-compare" # pragma clang diagnostic ignored "-Wstring-compare" #endif const char* zend_reg_name[] = { #if defined(__x86_64__) || defined(_M_X64) "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" #else "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" #endif }; #ifdef HAVE_GCC_GLOBAL_REGS # define GCC_GLOBAL_REGS 1 #else # define GCC_GLOBAL_REGS 0 #endif /* Simulate x86 fastcall */ #ifdef _WIN64 # define ZREG_FCARG1a ZREG_RCX # define ZREG_FCARG2a ZREG_RDX #elif defined(__x86_64__) # define ZREG_FCARG1a ZREG_RDI # define ZREG_FCARG2a ZREG_RSI #else # define ZREG_FCARG1a ZREG_RCX # define ZREG_FCARG2a ZREG_RDX #endif #if ZTS static size_t tsrm_ls_cache_tcb_offset = 0; static size_t tsrm_tls_index; static size_t tsrm_tls_offset; #endif /* By default avoid JITing inline handlers if it does not seem profitable due to lack of * type information. Disabling this option allows testing some JIT handlers in the * presence of try/catch blocks, which prevent SSA construction. */ #ifndef PROFITABILITY_CHECKS # define PROFITABILITY_CHECKS 1 #endif |.type EX, zend_execute_data, FP |.type OP, zend_op |.type ZVAL, zval |.actionlist dasm_actions |.globals zend_lb static void* dasm_labels[zend_lb_MAX]; |.section code, cold_code, jmp_table #define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) #define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) #define BP_JIT_IS 6 #define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) |.macro ADD_HYBRID_SPAD ||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | add r4, HYBRID_SPAD ||#endif |.endmacro |.macro SUB_HYBRID_SPAD ||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | sub r4, HYBRID_SPAD ||#endif |.endmacro |.macro LOAD_ADDR, reg, addr | .if X64 || if (IS_SIGNED_32BIT(addr)) { | mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 || } else { | mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 || } | .else | mov reg, ((ptrdiff_t)addr) | .endif |.endmacro |.macro LOAD_TSRM_CACHE, reg | .if X64WIN | gs | mov reg, aword [0x58] | mov reg, aword [reg+tsrm_tls_index] | mov reg, aword [reg+tsrm_tls_offset] | .elif WIN | fs | mov reg, aword [0x2c] | mov reg, aword [reg+tsrm_tls_index] | mov reg, aword [reg+tsrm_tls_offset] | .elif X64APPLE | gs || if (tsrm_ls_cache_tcb_offset) { | mov reg, aword [tsrm_ls_cache_tcb_offset] || } else { | mov reg, aword [tsrm_tls_index] | mov reg, aword [reg+tsrm_tls_offset] || } | .elif X64 | fs || if (tsrm_ls_cache_tcb_offset) { | mov reg, aword [tsrm_ls_cache_tcb_offset] || } else { | mov reg, [0x8] | mov reg, aword [reg+tsrm_tls_index] | mov reg, aword [reg+tsrm_tls_offset] || } | .else | gs || if (tsrm_ls_cache_tcb_offset) { | mov reg, aword [tsrm_ls_cache_tcb_offset] || } else { | mov reg, [0x4] | mov reg, aword [reg+tsrm_tls_index] | mov reg, aword [reg+tsrm_tls_offset] || } | .endif |.endmacro |.macro LOAD_ADDR_ZTS, reg, struct, field | .if ZTS | LOAD_TSRM_CACHE reg | lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] | .else | LOAD_ADDR reg, &struct.field | .endif |.endmacro |.macro ADDR_OP1, addr_ins, addr, tmp_reg | .if X64 || if (IS_SIGNED_32BIT(addr)) { | addr_ins ((ptrdiff_t)addr) || } else { | mov64 tmp_reg, ((ptrdiff_t)addr) | addr_ins tmp_reg || } | .else | addr_ins ((ptrdiff_t)addr) | .endif |.endmacro |.macro ADDR_OP2_2, addr_ins, op1, addr, tmp_reg | .if X64 || if (IS_SIGNED_32BIT(addr)) { | addr_ins op1, ((ptrdiff_t)addr) || } else { | mov64 tmp_reg, ((ptrdiff_t)addr) | addr_ins op1, tmp_reg || } | .else | addr_ins op1, ((ptrdiff_t)addr) | .endif |.endmacro |.macro PUSH_ADDR, addr, tmp_reg | ADDR_OP1 push, addr, tmp_reg |.endmacro |.macro PUSH_ADDR_ZTS, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE tmp_reg | lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] | push tmp_reg | .else | ADDR_OP1 push, &struct.field, tmp_reg | .endif |.endmacro |.macro MEM_OP1, mem_ins, prefix, addr, tmp_reg | .if X64 || if (IS_SIGNED_32BIT(addr)) { | mem_ins prefix [addr] || } else { | mov64 tmp_reg, ((ptrdiff_t)addr) | mem_ins prefix [tmp_reg] || } | .else | mem_ins prefix [addr] | .endif |.endmacro |.macro MEM_OP2_1, mem_ins, prefix, addr, op2, tmp_reg | .if X64 || if (IS_SIGNED_32BIT(addr)) { | mem_ins prefix [addr], op2 || } else { | mov64 tmp_reg, ((ptrdiff_t)addr) | mem_ins prefix [tmp_reg], op2 || } | .else | mem_ins prefix [addr], op2 | .endif |.endmacro |.macro MEM_OP2_2, mem_ins, op1, prefix, addr, tmp_reg | .if X64 || if (IS_SIGNED_32BIT(addr)) { | mem_ins op1, prefix [addr] || } else { | mov64 tmp_reg, ((ptrdiff_t)addr) | mem_ins op1, prefix [tmp_reg] || } | .else | mem_ins op1, prefix [addr] | .endif |.endmacro |.macro MEM_OP2_1_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg | .if ZTS | LOAD_TSRM_CACHE tmp_reg | mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2 | .else | MEM_OP2_1 mem_ins, prefix, &struct.field, op2, tmp_reg | .endif |.endmacro |.macro MEM_OP2_2_ZTS, mem_ins, op1, prefix, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE tmp_reg | mem_ins op1, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))] | .else | MEM_OP2_2 mem_ins, op1, prefix, &struct.field, tmp_reg | .endif |.endmacro |.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg | .if X64 || if (IS_SIGNED_32BIT(addr)) { | mem_ins op1, op2, prefix [addr] || } else { | mov64 tmp_reg, ((ptrdiff_t)addr) | mem_ins op1, op2, prefix [tmp_reg] || } | .else | mem_ins op1, op2, prefix [addr] | .endif |.endmacro |.macro LOAD_BASE_ADDR, reg, base, offset || if (offset) { | lea reg, qword [Ra(base)+offset] || } else { | mov reg, Ra(base) || } |.endmacro |.macro PUSH_BASE_ADDR, base, offset, tmp_reg || if (offset) { | lea tmp_reg, qword [Ra(base)+offset] | push tmp_reg || } else { | push Ra(base) || } |.endmacro |.macro EXT_CALL, func, tmp_reg | .if X64 || if (IS_32BIT(dasm_end) && IS_32BIT(func)) { | call qword &func || } else { | LOAD_ADDR tmp_reg, func | call tmp_reg || } | .else | call dword &func | .endif |.endmacro |.macro EXT_JMP, func, tmp_reg | .if X64 || if (IS_32BIT(dasm_end) && IS_32BIT(func)) { | jmp qword &func || } else { | LOAD_ADDR tmp_reg, func | jmp tmp_reg || } | .else | jmp dword &func | .endif |.endmacro |.macro SAVE_IP || if (GCC_GLOBAL_REGS) { | mov aword EX->opline, IP || } |.endmacro |.macro LOAD_IP || if (GCC_GLOBAL_REGS) { | mov IP, aword EX->opline || } |.endmacro |.macro LOAD_IP_ADDR, addr || if (GCC_GLOBAL_REGS) { | LOAD_ADDR IP, addr || } else { | ADDR_OP2_2 mov, aword EX->opline, addr, RX || } |.endmacro |.macro LOAD_IP_ADDR_ZTS, struct, field | .if ZTS || if (GCC_GLOBAL_REGS) { | LOAD_TSRM_CACHE IP | mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] || } else { | LOAD_TSRM_CACHE RX | lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] | mov aword EX->opline, RX || } | .else | LOAD_IP_ADDR &struct.field | .endif |.endmacro |.macro GET_IP, reg || if (GCC_GLOBAL_REGS) { | mov reg, IP || } else { | mov reg, aword EX->opline || } |.endmacro |.macro ADD_IP, val || if (GCC_GLOBAL_REGS) { | add IP, val || } else { | add aword EX->opline, val || } |.endmacro |.macro JMP_IP || if (GCC_GLOBAL_REGS) { | jmp aword [IP] || } else { | mov r0, aword EX:FCARG1a->opline | jmp aword [r0] || } |.endmacro /* In 64-bit build we compare only low 32-bits. * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full * comparison would require an additional load of 64-bit address into register. * This is not a problem at all, while JIT buffer size is less than 4GB. */ |.macro CMP_IP, addr || if (GCC_GLOBAL_REGS) { | cmp IPl, addr || } else { | cmp dword EX->opline, addr || } |.endmacro |.macro LOAD_ZVAL_ADDR, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { | LOAD_ADDR reg, Z_ZV(addr) || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr) || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro PUSH_ZVAL_ADDR, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | PUSH_ADDR Z_ZV(addr), tmp_reg || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro GET_Z_TYPE_INFO, reg, zv | mov reg, dword [zv+offsetof(zval,u1.type_info)] |.endmacro |.macro SET_Z_TYPE_INFO, zv, type | mov dword [zv+offsetof(zval,u1.type_info)], type |.endmacro |.macro GET_ZVAL_TYPE, reg, addr || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] |.endmacro |.macro GET_ZVAL_TYPE_INFO, reg, addr || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] |.endmacro |.macro SET_ZVAL_TYPE_INFO, addr, type || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type |.endmacro |.macro GET_Z_PTR, reg, zv | mov reg, aword [zv] |.endmacro |.macro SET_Z_PTR, zv, val | mov aword [zv], val |.endmacro |.macro GET_Z_W2, reg, zv | mov reg, dword [zv+4] |.endmacro |.macro SET_Z_W2, zv, reg | mov dword [zv+4], reg |.endmacro |.macro GET_ZVAL_PTR, reg, addr || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] |.endmacro |.macro SET_ZVAL_PTR, addr, val || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val |.endmacro |.macro GET_ZVAL_W2, reg, addr || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] |.endmacro |.macro SET_ZVAL_W2, addr, val || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val |.endmacro |.macro UNDEF_OPLINE_RESULT | mov r0, EX->opline | mov eax, dword OP:r0->result.var | SET_Z_TYPE_INFO FP + r0, IS_UNDEF |.endmacro |.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 || if (CAN_USE_AVX()) { | avx_ins op1, op2 || } else { | sse_ins op1, op2 || } |.endmacro |.macro SSE_OP, sse_ins, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | MEM_OP2_2 sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { | sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro SSE_AVX_OP, sse_ins, avx_ins, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { | .if X64 || if (IS_SIGNED_32BIT(Z_ZV(addr))) { | SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] || } else { | LOAD_ADDR r0, Z_ZV(addr) | SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [r0] || } | .else | SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { | SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro SSE_GET_LONG, reg, lval, tmp_reg || if (lval == 0) { || if (CAN_USE_AVX()) { | vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) || } else { | xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) || } || } else { |.if X64 || if (!IS_SIGNED_32BIT(lval)) { | mov64 Ra(tmp_reg), lval || } else { | mov Ra(tmp_reg), lval || } |.else | mov Ra(tmp_reg), lval |.endif || if (CAN_USE_AVX()) { | vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) | vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg) || } else { | xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) | cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg) || } || } |.endmacro |.macro SSE_GET_ZVAL_LVAL, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | SSE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { || if (CAN_USE_AVX()) { | vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) | vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else { | xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) | cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } || } else if (Z_MODE(addr) == IS_REG) { || if (CAN_USE_AVX()) { | vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) | vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) || } else { | xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) | cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) || } || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro SSE_GET_ZVAL_DVAL, reg, addr || if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { || if (Z_MODE(addr) == IS_CONST_ZVAL) { | .if X64 || if (IS_SIGNED_32BIT(Z_ZV(addr))) { | SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] || } else { | LOAD_ADDR r0, Z_ZV(addr) | SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] || } | .else | SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { | SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) || } else { || ZEND_UNREACHABLE(); || } || } |.endmacro |.macro SSE_MATH, opcode, reg, addr, tmp_reg || switch (opcode) { || case ZEND_ADD: | SSE_OP addsd, reg, addr, tmp_reg || break; || case ZEND_SUB: | SSE_OP subsd, reg, addr, tmp_reg || break; || case ZEND_MUL: | SSE_OP mulsd, reg, addr, tmp_reg || break; || case ZEND_DIV: | SSE_OP divsd, reg, addr, tmp_reg || break; || } |.endmacro |.macro SSE_MATH_REG, opcode, dst_reg, src_reg || switch (opcode) { || case ZEND_ADD: | addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) || break; || case ZEND_SUB: | subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) || break; || case ZEND_MUL: | mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) || break; || case ZEND_DIV: | divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) || break; || } |.endmacro |.macro SSE_SET_ZVAL_DVAL, addr, reg || if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { | SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) || } || } else { || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) || } |.endmacro |.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | MEM_OP3_3 avx_ins, xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { | avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg || switch (opcode) { || case ZEND_ADD: | AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg || break; || case ZEND_SUB: | AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg || break; || case ZEND_MUL: | AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg || break; || case ZEND_DIV: | AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg || break; || } |.endmacro |.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg || switch (opcode) { || case ZEND_ADD: | vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) || break; || case ZEND_SUB: | vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) || break; || case ZEND_MUL: | vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) || break; || case ZEND_DIV: | vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) || break; || } |.endmacro |.macro LONG_OP, long_ins, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { || if (reg != ZREG_R0) { | mov64 r0, Z_LVAL_P(Z_ZV(addr)) | long_ins Ra(reg), r0 || } else { | mov64 r1, Z_LVAL_P(Z_ZV(addr)) | long_ins Ra(reg), r1 || } || } else { | long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) || } | .else | long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) | .endif || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { | long_ins Ra(reg), Ra(Z_REG(addr)) || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval || if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { | long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval || } else if (Z_MODE(op1_addr) == IS_REG) { | long_ins Ra(Z_REG(op1_addr)), lval || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval || if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { | .if X64 || if (!IS_SIGNED_32BIT(lval)) { | mov64 r0, lval | long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 || } else { | long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval || } | .else | long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval | .endif || } else if (Z_MODE(op1_addr) == IS_REG) { | .if X64 || if (!IS_SIGNED_32BIT(lval)) { | mov64 r0, lval | long_ins Ra(Z_REG(op1_addr)), r0 || } else { | long_ins Ra(Z_REG(op1_addr)), lval || } | .else | long_ins Ra(Z_REG(op1_addr)), lval | .endif || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro GET_ZVAL_LVAL, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { || if (Z_LVAL_P(Z_ZV(addr)) == 0) { | xor Ra(reg), Ra(reg) || } else { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { | mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) || } else { | mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) || } | .else | mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) | .endif || } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] || } else if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { | mov Ra(reg), Ra(Z_REG(addr)) || } || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro LONG_MATH, opcode, reg, addr || switch (opcode) { || case ZEND_ADD: | LONG_OP add, reg, addr || break; || case ZEND_SUB: | LONG_OP sub, reg, addr || break; || case ZEND_MUL: | LONG_OP imul, reg, addr || break; || case ZEND_BW_OR: | LONG_OP or, reg, addr || break; || case ZEND_BW_AND: | LONG_OP and, reg, addr || break; || case ZEND_BW_XOR: | LONG_OP xor, reg, addr || break; || default: || ZEND_UNREACHABLE(); || } |.endmacro |.macro LONG_MATH_REG, opcode, dst_reg, src_reg || switch (opcode) { || case ZEND_ADD: | add dst_reg, src_reg || break; || case ZEND_SUB: | sub dst_reg, src_reg || break; || case ZEND_MUL: | imul dst_reg, src_reg || break; || case ZEND_BW_OR: | or dst_reg, src_reg || break; || case ZEND_BW_AND: | and dst_reg, src_reg || break; || case ZEND_BW_XOR: | xor dst_reg, src_reg || break; || default: || ZEND_UNREACHABLE(); || } |.endmacro |.macro SET_ZVAL_LVAL, addr, lval || if (Z_MODE(addr) == IS_REG) { | mov Ra(Z_REG(addr)), lval || } else { || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval || } |.endmacro |.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg || if (Z_TYPE_P(zv) > IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { || if (CAN_USE_AVX()) { | vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } else { | xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } | .if X64 || } else if (!IS_SIGNED_32BIT(zv)) { | mov64 Ra(tmp_reg), ((uintptr_t)zv) | SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] | .endif || } else { | SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] || } | SSE_SET_ZVAL_DVAL dst_addr, dst_reg || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; | SSE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0 | SSE_SET_ZVAL_DVAL dst_addr, dst_reg || } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { | xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) || } else { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { || if (Z_MODE(dst_addr) == IS_REG) { | mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) || } else { | mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) | SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) || } || } else { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) || } | .else | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) | .endif || } || } || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if (dst_def_info == MAY_BE_DOUBLE) { || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE || } || } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1< IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? || Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : ZREG_XMM0); || if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { || if (CAN_USE_AVX()) { | vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } else { | xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) || } | .if X64 || } else if (!IS_SIGNED_32BIT(zv)) { | mov64 Ra(tmp_reg), ((uintptr_t)zv) | SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] | .endif || } else { | SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] || } | SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 | SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { || if (Z_MODE(dst_addr) == IS_REG) { | SSE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0 | SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) || } else if (Z_MODE(res_addr) == IS_REG) { | SSE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0 | SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) || } else { | SSE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0 | SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 | SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 || } || } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { || if (Z_MODE(dst_addr) == IS_REG) { | xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) | SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) || } else { | xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) | SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) || } || } else { | .if X64 || if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { || if (Z_MODE(dst_addr) == IS_REG) { | mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) | SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) || } else if (Z_MODE(res_addr) == IS_REG) { | mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) | SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) || } else { | mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) | SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) || } || } else if (Z_MODE(dst_addr) == IS_REG) { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) || } else if (Z_MODE(res_addr) == IS_REG) { | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) || } else { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) || } | .else || if (Z_MODE(dst_addr) == IS_REG) { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) || } else if (Z_MODE(res_addr) == IS_REG) { | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) || } else { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) || } | .endif || } || } || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if (dst_def_info == MAY_BE_DOUBLE) { || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE || } || } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<1 || } | GC_ADDREF value_ptr_reg |1: || } |.endmacro |.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg || if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_NOT_REFCOUNTED type_flags_reg, >1 || } | add dword [value_ptr_reg], 2 |1: || } |.endmacro |.macro ZVAL_DEREF, reg, info || if (info & MAY_BE_REF) { | IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 | GET_Z_PTR reg, reg | add reg, offsetof(zend_reference, val) |1: || } |.endmacro |.macro SET_EX_OPLINE, op, tmp_reg || if (op == last_valid_opline) { || zend_jit_use_last_valid_opline(); | SAVE_IP || } else { | ADDR_OP2_2 mov, aword EX->opline, op, tmp_reg || if (!GCC_GLOBAL_REGS) { || zend_jit_reset_last_valid_opline(); || } || } |.endmacro // zval should be in FCARG1a |.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a || do { || if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_INDIRECT))) { || zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); || if (type == IS_STRING && !ZEND_DEBUG) { | EXT_CALL _efree, r0 || break; || } else if (type == IS_ARRAY) { || if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { || if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { | SET_EX_OPLINE opline, r0 || } | EXT_CALL zend_array_destroy, r0 || } else { | EXT_CALL zend_jit_array_free, r0 || } || break; || } else if (type == IS_OBJECT) { || if (opline) { | SET_EX_OPLINE opline, r0 || } | EXT_CALL zend_objects_store_del, r0 || break; || } || } || if (opline) { | SET_EX_OPLINE opline, r0 || } | EXT_CALL rc_dtor_func, r0 || } while(0); |.endmacro |.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline || if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { || if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { || if (cold) { | IF_ZVAL_REFCOUNTED addr, >1 |.cold_code |1: || } else { | IF_NOT_ZVAL_REFCOUNTED addr, >4 || } || } | // if (!Z_DELREF_P(cv)) { | GET_ZVAL_PTR FCARG1a, addr | GC_DELREF FCARG1a || if (RC_MAY_BE_1(op_info)) { || if (RC_MAY_BE_N(op_info)) { || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { | jnz >3 || } else { | jnz >4 || } || } | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op_info, opline || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { | jmp >4 || } |3: || } || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { || if ((op_info) & MAY_BE_REF) { || zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val)); | IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 | IF_NOT_ZVAL_COLLECTABLE ref_addr, >4 | GET_ZVAL_PTR FCARG1a, ref_addr |1: || } | IF_GC_MAY_NOT_LEAK FCARG1a, >4 | // gc_possible_root(Z_COUNTED_P(z)) | EXT_CALL gc_possible_root, r0 || } || if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { | jmp >4 |.code || } |4: || } |.endmacro |.macro FREE_OP, op_type, op, op_info, cold, opline || if (op_type & (IS_VAR|IS_TMP_VAR)) { | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline || } |.endmacro |.macro SEPARATE_ARRAY, addr, op_info, cold || if (RC_MAY_BE_N(op_info)) { || if (Z_REG(addr) != ZREG_FP) { | GET_ZVAL_LVAL ZREG_R0, addr || if (RC_MAY_BE_1(op_info)) { | cmp dword [r0], 1 // if (GC_REFCOUNT() > 1) | jbe >2 || } || if (Z_REG(addr) != ZREG_FCARG1a || Z_OFFSET(addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, addr || } | EXT_CALL zend_jit_zval_array_dup, r0 |2: | mov FCARG1a, r0 || } else { | GET_ZVAL_LVAL ZREG_FCARG1a, addr || if (RC_MAY_BE_1(op_info)) { | cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1) || if (cold) { | ja >1 |.cold_code |1: || } else { | jbe >2 || } || } | IF_NOT_ZVAL_REFCOUNTED addr, >1 | GC_DELREF FCARG1a |1: | EXT_CALL zend_array_dup, r0 | SET_ZVAL_PTR addr, r0 | SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX | mov FCARG1a, r0 || if (RC_MAY_BE_1(op_info)) { || if (cold) { | jmp >2 |.code || } || } |2: || } || } else { | GET_ZVAL_LVAL ZREG_FCARG1a, addr || } |.endmacro |.macro EFREE_REG_REFERENCE ||#if ZEND_DEBUG | xor FCARG2a, FCARG2a // filename | .if X64WIN | xor CARG3d, CARG3d // lineno | xor CARG4, CARG4 | mov aword A5, 0 | EXT_CALL _efree, r0 | .elif X64 | xor CARG3d, CARG3d // lineno | xor CARG4, CARG4 | xor CARG5, CARG5 | EXT_CALL _efree, r0 | .else | sub r4, 4 | push 0 | push 0 | push 0 // lineno | EXT_CALL _efree, r0 | add r4, 4 | .endif ||#else ||#ifdef HAVE_BUILTIN_CONSTANT_P | EXT_CALL _efree_32, r0 ||#else | EXT_CALL _efree, r0 ||#endif ||#endif |.endmacro |.macro EFREE_REFERENCE, ptr | mov FCARG1a, ptr | EFREE_REG_REFERENCE |.endmacro |.macro EMALLOC, size, op_array, opline ||#if ZEND_DEBUG || const char *filename = op_array->filename ? op_array->filename->val : NULL; | mov FCARG1a, size | LOAD_ADDR FCARG2a, filename | .if X64WIN | mov CARG3d, opline->lineno | xor CARG4, CARG4 | mov aword A5, 0 | EXT_CALL _emalloc, r0 | .elif X64 | mov CARG3d, opline->lineno | xor CARG4, CARG4 | xor CARG5, CARG5 | EXT_CALL _emalloc, r0 | .else | sub r4, 4 | push 0 | push 0 | push opline->lineno | EXT_CALL _emalloc, r0 | add r4, 4 | .endif ||#else ||#ifdef HAVE_BUILTIN_CONSTANT_P || if (size > 24 && size <= 32) { | EXT_CALL _emalloc_32, r0 || } else { | mov FCARG1a, size | EXT_CALL _emalloc, r0 || } ||#else | mov FCARG1a, size | EXT_CALL _emalloc, r0 ||#endif ||#endif |.endmacro |.macro OBJ_RELEASE, reg, exit_label | GC_DELREF Ra(reg) | jne >1 | // zend_objects_store_del(obj); || if (reg != ZREG_FCARG1a) { | mov FCARG1a, Ra(reg) || } | EXT_CALL zend_objects_store_del, r0 | jmp exit_label |1: | IF_GC_MAY_NOT_LEAK Ra(reg), >1 | // gc_possible_root(obj) || if (reg != ZREG_FCARG1a) { | mov FCARG1a, Ra(reg) || } | EXT_CALL gc_possible_root, r0 |1: |.endmacro |.macro UNDEFINED_OFFSET, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); | call ->undefined_offset_ex || } else { | SET_EX_OPLINE opline, r0 | call ->undefined_offset || } |.endmacro |.macro UNDEFINED_INDEX, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); | call ->undefined_index_ex || } else { | SET_EX_OPLINE opline, r0 | call ->undefined_index || } |.endmacro |.macro CANNOT_ADD_ELEMENT, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); | call ->cannot_add_element_ex || } else { | SET_EX_OPLINE opline, r0 | call ->cannot_add_element || } |.endmacro static bool reuse_ip = 0; static bool delayed_call_chain = 0; static uint32_t delayed_call_level = 0; static const zend_op *last_valid_opline = NULL; static bool use_last_vald_opline = 0; static bool track_last_valid_opline = 0; static int jit_return_label = -1; static uint32_t current_trace_num = 0; static uint32_t allowed_opt_flags = 0; static void zend_jit_track_last_valid_opline(void) { use_last_vald_opline = 0; track_last_valid_opline = 1; } static void zend_jit_use_last_valid_opline(void) { if (track_last_valid_opline) { use_last_vald_opline = 1; track_last_valid_opline = 0; } } static bool zend_jit_trace_uses_initial_ip(void) { return use_last_vald_opline; } static void zend_jit_set_last_valid_opline(const zend_op *target_opline) { if (!reuse_ip) { track_last_valid_opline = 0; last_valid_opline = target_opline; } } static void zend_jit_reset_last_valid_opline(void) { track_last_valid_opline = 0; last_valid_opline = NULL; } static void zend_jit_start_reuse_ip(void) { zend_jit_reset_last_valid_opline(); reuse_ip = 1; } static int zend_jit_reuse_ip(dasm_State **Dst) { if (!reuse_ip) { zend_jit_start_reuse_ip(); | // call = EX(call); | mov RX, EX->call } return 1; } static void zend_jit_stop_reuse_ip(void) { reuse_ip = 0; } /* bit helpers */ /* from http://aggregate.org/MAGIC/ */ static uint32_t ones32(uint32_t x) { x -= ((x >> 1) & 0x55555555); x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); x = (((x >> 4) + x) & 0x0f0f0f0f); x += (x >> 8); x += (x >> 16); return x & 0x0000003f; } static uint32_t floor_log2(uint32_t x) { ZEND_ASSERT(x != 0); x |= (x >> 1); x |= (x >> 2); x |= (x >> 4); x |= (x >> 8); x |= (x >> 16); return ones32(x) - 1; } static bool is_power_of_two(uint32_t x) { return !(x & (x - 1)) && x != 0; } static bool has_concrete_type(uint32_t value_type) { return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); } static uint32_t concrete_type(uint32_t value_type) { return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); } static inline bool is_signed(double d) { return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; } static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: | SAVE_IP | //EG(vm_interrupt) = 0; | MEM_OP2_1_ZTS mov, byte, executor_globals, vm_interrupt, 0, r0 | //if (EG(timed_out)) { | MEM_OP2_1_ZTS cmp, byte, executor_globals, timed_out, 0, r0 | je >1 | //zend_timeout(); | EXT_CALL zend_timeout, r0 |1: | //} else if (zend_interrupt_function) { if (zend_interrupt_function) { | //zend_interrupt_function(execute_data); |.if X64 | mov CARG1, FP | EXT_CALL zend_interrupt_function, r0 |.else | mov aword A1, FP | EXT_CALL zend_interrupt_function, r0 |.endif | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 | LOAD_IP } | //ZEND_VM_CONTINUE() if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | JMP_IP } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment | JMP_IP } else { | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 1 // ZEND_VM_ENTER | ret } return 1; } static int zend_jit_exception_handler_stub(dasm_State **Dst) { |->exception_handler: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { const void *handler = zend_get_opcode_handler_func(EG(exception_op)); | ADD_HYBRID_SPAD | EXT_CALL handler, r0 | JMP_IP } else { const void *handler = EG(exception_op)->handler; if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment | EXT_JMP handler, r0 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | mov FCARG1a, FP | EXT_CALL handler, r0 | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | test eax, eax | jl >1 | mov r0, 1 // ZEND_VM_ENTER |1: | ret } else { | mov FCARG1a, FP | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | EXT_JMP handler, r0 } } return 1; } static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) { |->exception_handler_undef: | MEM_OP2_2_ZTS mov, r0, aword, executor_globals, opline_before_exception, r0 | test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR) | jnz >1 | mov eax, dword OP:r0->result.var | SET_Z_TYPE_INFO FP + r0, IS_UNDEF |1: | jmp ->exception_handler return 1; } static int zend_jit_leave_function_stub(dasm_State **Dst) { |->leave_function_handler: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | test FCARG1d, ZEND_CALL_TOP | jnz >1 | EXT_CALL zend_jit_leave_nested_func_helper, r0 | ADD_HYBRID_SPAD | JMP_IP |1: | EXT_CALL zend_jit_leave_top_func_helper, r0 | ADD_HYBRID_SPAD | JMP_IP } else { if (GCC_GLOBAL_REGS) { | add r4, SPAD } else { | mov FCARG2a, FP | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD } | test FCARG1d, ZEND_CALL_TOP | jnz >1 | EXT_JMP zend_jit_leave_nested_func_helper, r0 |1: | EXT_JMP zend_jit_leave_top_func_helper, r0 } return 1; } static int zend_jit_leave_throw_stub(dasm_State **Dst) { |->leave_throw_handler: | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { if (GCC_GLOBAL_REGS) { | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION | je >5 | // EG(opline_before_exception) = opline; | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0 |5: | // opline = EG(exception_op); | LOAD_IP_ADDR_ZTS executor_globals, exception_op | // HANDLE_EXCEPTION() | jmp ->exception_handler } else { | GET_IP FCARG1a | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION | je >5 | // EG(opline_before_exception) = opline; | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, FCARG1a, r0 |5: | // opline = EG(exception_op); | LOAD_IP_ADDR_ZTS executor_globals, exception_op | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 2 // ZEND_VM_LEAVE | ret } return 1; } static int zend_jit_icall_throw_stub(dasm_State **Dst) { |->icall_throw_handler: | // zend_rethrow_exception(zend_execute_data *execute_data) | mov IP, aword EX->opline | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION | je >1 | // EG(opline_before_exception) = opline; | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0 |1: | // opline = EG(exception_op); | LOAD_IP_ADDR_ZTS executor_globals, exception_op || if (GCC_GLOBAL_REGS) { | mov aword EX->opline, IP || } | // HANDLE_EXCEPTION() | jmp ->exception_handler return 1; } static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) { |->throw_cannot_pass_by_ref: | mov r0, EX->opline | mov ecx, dword OP:r0->result.var | SET_Z_TYPE_INFO RX+r1, IS_UNDEF | // last EX(call) frame may be delayed | cmp RX, EX->call | je >1 | mov r1, EX->call | mov EX:RX->prev_execute_data, r1 | mov EX->call, RX |1: | mov RX, r0 | mov FCARG1d, dword OP:r0->op2.num | EXT_CALL zend_cannot_pass_by_reference, r0 | cmp byte OP:RX->op1_type, IS_TMP_VAR | jne >9 | mov eax, dword OP:RX->op1.var | add r0, FP | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL |9: | jmp ->exception_handler return 1; } static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) { |->undefined_offset_ex: | SAVE_IP | jmp ->undefined_offset return 1; } static int zend_jit_undefined_offset_stub(dasm_State **Dst) { |->undefined_offset: |.if X64WIN | sub r4, 0x28 |.elif X64 | sub r4, 8 |.else | sub r4, 12 |.endif | mov r0, EX->opline | mov ecx, dword OP:r0->result.var | cmp byte OP:r0->op2_type, IS_CONST | SET_Z_TYPE_INFO FP + r1, IS_NULL | jne >2 |.if X64 | movsxd r1, dword OP:r0->op2.constant | add r0, r1 |.else | mov r0, aword OP:r0->op2.zv |.endif | jmp >3 |2: | mov eax, dword OP:r0->op2.var | add r0, FP |3: |.if X64WIN | mov CARG1, E_WARNING | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT | mov CARG3, aword [r0] | EXT_CALL zend_error, r0 | add r4, 0x28 // stack alignment |.elif X64 | mov CARG1, E_WARNING | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT | mov CARG3, aword [r0] | EXT_CALL zend_error, r0 | add r4, 8 // stack alignment |.else | sub r4, 4 | push aword [r0] | push "Undefined array key " ZEND_LONG_FMT | push E_WARNING | EXT_CALL zend_error, r0 | add r4, 28 |.endif | ret return 1; } static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) { |->undefined_index_ex: | SAVE_IP | jmp ->undefined_index return 1; } static int zend_jit_undefined_index_stub(dasm_State **Dst) { |->undefined_index: |.if X64WIN | sub r4, 0x28 |.elif X64 | sub r4, 8 |.else | sub r4, 12 |.endif | mov r0, EX->opline | mov ecx, dword OP:r0->result.var | cmp byte OP:r0->op2_type, IS_CONST | SET_Z_TYPE_INFO FP + r1, IS_NULL | jne >2 |.if X64 | movsxd r1, dword OP:r0->op2.constant | add r0, r1 |.else | mov r0, aword OP:r0->op2.zv |.endif | jmp >3 |2: | mov eax, dword OP:r0->op2.var | add r0, FP |3: |.if X64WIN | mov CARG1, E_WARNING | LOAD_ADDR CARG2, "Undefined array key \"%s\"" | mov CARG3, aword [r0] | add CARG3, offsetof(zend_string, val) | EXT_CALL zend_error, r0 | add r4, 0x28 |.elif X64 | mov CARG1, E_WARNING | LOAD_ADDR CARG2, "Undefined array key \"%s\"" | mov CARG3, aword [r0] | add CARG3, offsetof(zend_string, val) | EXT_CALL zend_error, r0 | add r4, 8 |.else | sub r4, 4 | mov r0, aword [r0] | add r0, offsetof(zend_string, val) | push r0 | push "Undefined array key \"%s\"" | push E_WARNING | EXT_CALL zend_error, r0 | add r4, 28 |.endif | ret return 1; } static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) { |->cannot_add_element_ex: | SAVE_IP | jmp ->cannot_add_element return 1; } static int zend_jit_cannot_add_element_stub(dasm_State **Dst) { |->cannot_add_element: |.if X64WIN | sub r4, 0x28 |.elif X64 | sub r4, 8 |.else | sub r4, 12 |.endif | mov r0, EX->opline | cmp byte OP:r0->result_type, IS_UNUSED | jz >1 | mov eax, dword OP:r0->result.var | SET_Z_TYPE_INFO FP + r0, IS_NULL |1: |.if X64WIN | xor CARG1, CARG1 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" | EXT_CALL zend_throw_error, r0 | add r4, 0x28 |.elif X64 | xor CARG1, CARG1 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" | EXT_CALL zend_throw_error, r0 | add r4, 8 |.else | sub r4, 8 | push "Cannot add element to the array as the next element is already occupied" | push 0 | EXT_CALL zend_throw_error, r0 | add r4, 28 |.endif | ret return 1; } static int zend_jit_undefined_function_stub(dasm_State **Dst) { |->undefined_function: | mov r0, aword EX->opline |.if X64 | xor CARG1, CARG1 | LOAD_ADDR CARG2, "Call to undefined function %s()" | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)] | mov CARG3, aword [r0 + CARG3] | add CARG3, offsetof(zend_string, val) | EXT_CALL zend_throw_error, r0 |.else | mov r0, aword [r0 + offsetof(zend_op, op2.zv)] | mov r0, aword [r0] | add r0, offsetof(zend_string, val) | mov aword A3, r0 | mov aword A2, "Call to undefined function %s()" | mov aword A1, 0 | EXT_CALL zend_throw_error, r0 |.endif | jmp ->exception_handler return 1; } static int zend_jit_negative_shift_stub(dasm_State **Dst) { |->negative_shift: | UNDEF_OPLINE_RESULT |.if X64 |.if WIN | LOAD_ADDR CARG1, &zend_ce_arithmetic_error | mov CARG1, aword [CARG1] |.else | LOAD_ADDR CARG1, zend_ce_arithmetic_error |.endif | LOAD_ADDR CARG2, "Bit shift by negative number" | EXT_CALL zend_throw_error, r0 |.else | sub r4, 8 | push "Bit shift by negative number" |.if WIN | LOAD_ADDR r0, &zend_ce_arithmetic_error | push aword [r0] |.else | PUSH_ADDR zend_ce_arithmetic_error, r0 |.endif | EXT_CALL zend_throw_error, r0 | add r4, 16 |.endif | jmp ->exception_handler return 1; } static int zend_jit_mod_by_zero_stub(dasm_State **Dst) { |->mod_by_zero: | UNDEF_OPLINE_RESULT |.if X64 |.if WIN | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error | mov CARG1, aword [CARG1] |.else | LOAD_ADDR CARG1, zend_ce_division_by_zero_error |.endif | LOAD_ADDR CARG2, "Modulo by zero" | EXT_CALL zend_throw_error, r0 |.else | sub r4, 8 | push "Modulo by zero" |.if WIN | LOAD_ADDR r0, &zend_ce_division_by_zero_error | push aword [r0] |.else | PUSH_ADDR zend_ce_division_by_zero_error, r0 |.endif | EXT_CALL zend_throw_error, r0 | add r4, 16 |.endif | jmp ->exception_handler return 1; } static int zend_jit_invalid_this_stub(dasm_State **Dst) { |->invalid_this: | UNDEF_OPLINE_RESULT |.if X64 | xor CARG1, CARG1 | LOAD_ADDR CARG2, "Using $this when not in object context" | EXT_CALL zend_throw_error, r0 |.else | sub r4, 8 | push "Using $this when not in object context" | push 0 | EXT_CALL zend_throw_error, r0 | add r4, 16 |.endif | jmp ->exception_handler return 1; } static int zend_jit_double_one_stub(dasm_State **Dst) { |->one: |.dword 0, 0x3ff00000 return 1; } static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { return 1; } |->hybrid_runtime_jit: | EXT_CALL zend_runtime_jit, r0 | JMP_IP return 1; } static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { return 1; } |->hybrid_profile_jit: | // ++zend_jit_profile_counter; | .if X64 | LOAD_ADDR r0, &zend_jit_profile_counter | inc aword [r0] | .else | inc aword [&zend_jit_profile_counter] | .endif | // op_array = (zend_op_array*)EX(func); | mov r0, EX->func | // run_time_cache = EX(run_time_cache); | mov r2, EX->run_time_cache | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | // ++ZEND_COUNTER_INFO(op_array) | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() | jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)] return 1; } static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { return 1; } |->hybrid_hot_code: | mov word [r2], ZEND_JIT_COUNTER_INIT | mov FCARG1a, FP | GET_IP FCARG2a | EXT_CALL zend_jit_hot_func, r0 | JMP_IP return 1; } /* * This code is based Mike Pall's "Hashed profile counters" idea, implemented * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual * property disclosure and research opportunities" email * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html * * In addition we use a variation of Knuth's multiplicative hash function * described at https://code.i-harness.com/en/q/a21ce * * uint64_t hash(uint64_t x) { * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; * x = x ^ (x >> 31); * return x; * } * * uint_32_t hash(uint32_t x) { * x = ((x >> 16) ^ x) * 0x45d9f3b; * x = ((x >> 16) ^ x) * 0x45d9f3b; * x = (x >> 16) ^ x; * return x; * } * */ static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) { | mov r0, EX->func | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] | sub word [r2], cost | jle ->hybrid_hot_code | GET_IP r2 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] | // divide by sizeof(zend_op) | .if X64 || ZEND_ASSERT(sizeof(zend_op) == 32); | sar r2, 2 | .else || ZEND_ASSERT(sizeof(zend_op) == 28); | imul r2, 0xb6db6db7 | .endif | .if X64 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] | .else | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] | .endif return 1; } static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { return 1; } |->hybrid_func_hot_counter: return zend_jit_hybrid_hot_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); } static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { return 1; } |->hybrid_loop_hot_counter: return zend_jit_hybrid_hot_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); } static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { return 1; } |->hybrid_hot_trace: | mov word [r2], ZEND_JIT_COUNTER_INIT | mov FCARG1a, FP | GET_IP FCARG2a | EXT_CALL zend_jit_trace_hot_root, r0 | test eax, eax // TODO : remove this check at least for HYBRID VM ??? | jl >1 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 | LOAD_IP | JMP_IP |1: | EXT_JMP zend_jit_halt_op->handler, r0 return 1; } static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) { | mov r0, EX->func | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)] | sub word [r2], cost | jle ->hybrid_hot_trace | jmp aword [IP + r1] return 1; } static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { return 1; } |->hybrid_func_trace_counter: return zend_jit_hybrid_trace_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); } static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { return 1; } |->hybrid_ret_trace_counter: return zend_jit_hybrid_trace_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); } static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { return 1; } |->hybrid_loop_trace_counter: return zend_jit_hybrid_trace_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); } static int zend_jit_trace_halt_stub(dasm_State **Dst) { |->trace_halt: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | EXT_JMP zend_jit_halt_op->handler, r0 } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment | ret // PC must be zero } else { | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, -1 // ZEND_VM_RETURN | ret } return 1; } static int zend_jit_trace_exit_stub(dasm_State **Dst) { |->trace_exit: | | // Save CPU registers |.if X64 | sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */ | mov aword [r4+15*8], r15 | mov aword [r4+11*8], r11 | mov aword [r4+10*8], r10 | mov aword [r4+9*8], r9 | mov aword [r4+8*8], r8 | mov aword [r4+7*8], rdi | mov aword [r4+6*8], rsi | mov aword [r4+2*8], rdx | mov aword [r4+1*8], rcx | mov aword [r4+0*8], rax | mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP | mov FCARG2a, r4 | movsd qword [r4+16*8+15*8], xmm15 | movsd qword [r4+16*8+14*8], xmm14 | movsd qword [r4+16*8+13*8], xmm13 | movsd qword [r4+16*8+12*8], xmm12 | movsd qword [r4+16*8+11*8], xmm11 | movsd qword [r4+16*8+10*8], xmm10 | movsd qword [r4+16*8+9*8], xmm9 | movsd qword [r4+16*8+8*8], xmm8 | movsd qword [r4+16*8+7*8], xmm7 | movsd qword [r4+16*8+6*8], xmm6 | movsd qword [r4+16*8+5*8], xmm5 | movsd qword [r4+16*8+4*8], xmm4 | movsd qword [r4+16*8+3*8], xmm3 | movsd qword [r4+16*8+2*8], xmm2 | movsd qword [r4+16*8+1*8], xmm1 | movsd qword [r4+16*8+0*8], xmm0 |.if X64WIN | sub r4, 32 /* shadow space */ |.endif |.else | sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */ | mov aword [r4+7*4], edi | mov aword [r4+2*4], edx | mov aword [r4+1*4], ecx | mov aword [r4+0*4], eax | mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP | mov FCARG2a, r4 | movsd qword [r4+8*4+7*8], xmm7 | movsd qword [r4+8*4+6*8], xmm6 | movsd qword [r4+8*4+5*8], xmm5 | movsd qword [r4+8*4+4*8], xmm4 | movsd qword [r4+8*4+3*8], xmm3 | movsd qword [r4+8*4+2*8], xmm2 | movsd qword [r4+8*4+1*8], xmm1 | movsd qword [r4+8*4+0*8], xmm0 |.endif | | // EX(opline) = opline | SAVE_IP | // zend_jit_trace_exit(trace_num, exit_num) | EXT_CALL zend_jit_trace_exit, r0 |.if X64WIN | add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */ |.elif X64 | add r4, 16*8+16*8 /* CPU regs + SSE regs */ |.else | add r4, 8*4+8*8 /* CPU regs + SSE regs */ |.endif | test eax, eax | jne >1 | // execute_data = EG(current_execute_data) | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 | // opline = EX(opline) | LOAD_IP if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | JMP_IP } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment | JMP_IP } else { | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 1 // ZEND_VM_ENTER | ret } |1: | jl ->trace_halt | // execute_data = EG(current_execute_data) | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 | // opline = EX(opline) | LOAD_IP | // check for interrupt (try to avoid this ???) | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 | jne ->interrupt_handler if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] | jmp aword [IP + r0] } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] | jmp aword [IP + r0] } else { | mov IP, aword EX->opline | mov FCARG1a, FP | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] | call aword [IP + r0] | test eax, eax | jl ->trace_halt | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 1 // ZEND_VM_ENTER | ret } return 1; } static int zend_jit_trace_escape_stub(dasm_State **Dst) { |->trace_escape: | if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | JMP_IP } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment | JMP_IP } else { | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 1 // ZEND_VM_ENTER | ret } return 1; } /* Keep 32 exit points in a single code block */ #define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes #define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) { uint32_t i; for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { | push byte i | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1 } | push byte i |// 1: | add aword [r4], n | jmp ->trace_exit return 1; } #ifdef CONTEXT_THREADED_JIT static int zend_jit_context_threaded_call_stub(dasm_State **Dst) { |->context_threaded_call: | pop r0 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | jmp aword [IP] } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment | jmp aword [IP] } else { ZEND_UNREACHABLE(); // TODO: context threading can't work without GLOBAL REGS because we have to change // the value of execute_data in execute_ex() | mov FCARG1a, FP | mov r0, aword [FP] | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | jmp aword [r0] } return 1; } #endif static int zend_jit_assign_to_variable(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_use_addr, zend_jit_addr var_addr, uint32_t var_info, uint32_t var_def_info, zend_uchar val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr, bool check_exception); static int zend_jit_assign_const_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_const: |.if X64WIN | sub r4, 0x28 |.elif X64 | sub r4, 8 |.else | sub r4, 12 |.endif if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_CONST, val_addr, val_info, 0, 0)) { return 0; } |.if X64WIN | add r4, 0x28 |.elif X64 | add r4, 8 |.else | add r4, 12 |.endif | ret return 1; } static int zend_jit_assign_tmp_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_tmp: |.if X64WIN | sub r4, 0x28 |.elif X64 | sub r4, 8 |.else | sub r4, 12 |.endif if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_TMP_VAR, val_addr, val_info, 0, 0)) { return 0; } |.if X64WIN | add r4, 0x28 |.elif X64 | add r4, 8 |.else | add r4, 12 |.endif | ret return 1; } static int zend_jit_assign_var_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; |->assign_var: |.if X64WIN | sub r4, 0x28 |.elif X64 | sub r4, 8 |.else | sub r4, 12 |.endif if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_VAR, val_addr, val_info, 0, 0)) { return 0; } |.if X64WIN | add r4, 0x28 |.elif X64 | add r4, 8 |.else | add r4, 12 |.endif | ret return 1; } static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; |->assign_cv_noref: |.if X64WIN | sub r4, 0x28 |.elif X64 | sub r4, 8 |.else | sub r4, 12 |.endif if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_CV, val_addr, val_info, 0, 0)) { return 0; } |.if X64WIN | add r4, 0x28 |.elif X64 | add r4, 8 |.else | add r4, 12 |.endif | ret return 1; } static int zend_jit_assign_cv_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; |->assign_cv: |.if X64WIN | sub r4, 0x28 |.elif X64 | sub r4, 8 |.else | sub r4, 12 |.endif if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_CV, val_addr, val_info, 0, 0)) { return 0; } |.if X64WIN | add r4, 0x28 |.elif X64 | add r4, 8 |.else | add r4, 12 |.endif | ret return 1; } static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler), JIT_STUB(exception_handler), JIT_STUB(exception_handler_undef), JIT_STUB(leave_function), JIT_STUB(leave_throw), JIT_STUB(icall_throw), JIT_STUB(throw_cannot_pass_by_ref), JIT_STUB(undefined_offset), JIT_STUB(undefined_index), JIT_STUB(cannot_add_element), JIT_STUB(undefined_offset_ex), JIT_STUB(undefined_index_ex), JIT_STUB(cannot_add_element_ex), JIT_STUB(undefined_function), JIT_STUB(negative_shift), JIT_STUB(mod_by_zero), JIT_STUB(invalid_this), JIT_STUB(trace_halt), JIT_STUB(trace_exit), JIT_STUB(trace_escape), JIT_STUB(hybrid_runtime_jit), JIT_STUB(hybrid_profile_jit), JIT_STUB(hybrid_hot_code), JIT_STUB(hybrid_func_hot_counter), JIT_STUB(hybrid_loop_hot_counter), JIT_STUB(hybrid_hot_trace), JIT_STUB(hybrid_func_trace_counter), JIT_STUB(hybrid_ret_trace_counter), JIT_STUB(hybrid_loop_trace_counter), JIT_STUB(assign_const), JIT_STUB(assign_tmp), JIT_STUB(assign_var), JIT_STUB(assign_cv_noref), JIT_STUB(assign_cv), JIT_STUB(double_one), #ifdef CONTEXT_THREADED_JIT JIT_STUB(context_threaded_call), #endif }; #if ZTS && defined(ZEND_WIN32) extern uint32_t _tls_index; extern char *_tls_start; extern char *_tls_end; #endif static int zend_jit_setup(void) { if (!zend_cpu_supports_sse2()) { zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); return FAILURE; } allowed_opt_flags = 0; if (zend_cpu_supports_avx()) { allowed_opt_flags |= ZEND_JIT_CPU_AVX; } #if ZTS # ifdef _WIN64 tsrm_tls_index = _tls_index * sizeof(void*); /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ /* Probably, it might be better solution */ do { void ***tls_mem = ((void**)__readgsqword(0x58))[_tls_index]; void *val = _tsrm_ls_cache; size_t offset = 0; size_t size = (char*)&_tls_end - (char*)&_tls_start; while (offset < size) { if (*tls_mem == val) { tsrm_tls_offset = offset; break; } tls_mem++; offset += sizeof(void*); } if (offset >= size) { // TODO: error message ??? return FAILURE; } } while(0); # elif ZEND_WIN32 tsrm_tls_index = _tls_index * sizeof(void*); /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ /* Probably, it might be better solution */ do { void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; void *val = _tsrm_ls_cache; size_t offset = 0; size_t size = (char*)&_tls_end - (char*)&_tls_start; while (offset < size) { if (*tls_mem == val) { tsrm_tls_offset = offset; break; } tls_mem++; offset += sizeof(void*); } if (offset >= size) { // TODO: error message ??? return FAILURE; } } while(0); # elif defined(__APPLE__) && defined(__x86_64__) tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); if (tsrm_ls_cache_tcb_offset == 0) { size_t *ti; __asm__( "leaq __tsrm_ls_cache(%%rip),%0" : "=r" (ti)); tsrm_tls_offset = ti[2]; tsrm_tls_index = ti[1] * 8; } # elif defined(__GNUC__) && defined(__x86_64__) tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); if (tsrm_ls_cache_tcb_offset == 0) { #if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) size_t ret; asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" : "=r" (ret)); tsrm_ls_cache_tcb_offset = ret; #else size_t *ti; __asm__( "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" : "=a" (ti)); tsrm_tls_offset = ti[1]; tsrm_tls_index = ti[0] * 16; #endif } # elif defined(__GNUC__) && defined(__i386__) tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); if (tsrm_ls_cache_tcb_offset == 0) { #if !defined(__FreeBSD__) size_t ret; asm ("leal _tsrm_ls_cache@ntpoff,%0\n" : "=a" (ret)); tsrm_ls_cache_tcb_offset = ret; #else size_t *ti, _ebx, _ecx, _edx; __asm__( "call 1f\n" ".subsection 1\n" "1:\tmovl (%%esp), %%ebx\n\t" "ret\n" ".previous\n\t" "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" "call ___tls_get_addr@plt\n\t" "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); tsrm_tls_offset = ti[1]; tsrm_tls_index = ti[0] * 8; #endif } # endif #endif return SUCCESS; } static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) { | int3 return 1; } static int zend_jit_align_func(dasm_State **Dst) { reuse_ip = 0; delayed_call_chain = 0; last_valid_opline = NULL; use_last_vald_opline = 0; track_last_valid_opline = 0; jit_return_label = -1; |.align 16 return 1; } static int zend_jit_prologue(dasm_State **Dst) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | SUB_HYBRID_SPAD } else if (GCC_GLOBAL_REGS) { | sub r4, SPAD // stack alignment } else { | sub r4, NR_SPAD // stack alignment | mov aword T2, FP // save FP | mov aword T3, RX // save IP | mov FP, FCARG1a } return 1; } static int zend_jit_label(dasm_State **Dst, unsigned int label) { |=>label: return 1; } static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) { | // call->prev_execute_data = EX(call); if (call_level == 1) { | mov aword EX:RX->prev_execute_data, 0 } else { | mov r0, EX->call | mov EX:RX->prev_execute_data, r0 } | // EX(call) = call; | mov EX->call, RX delayed_call_chain = 0; return 1; } static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) { if (last_valid_opline == opline) { zend_jit_use_last_valid_opline(); } else if (GCC_GLOBAL_REGS && last_valid_opline) { zend_jit_use_last_valid_opline(); | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); } else { | LOAD_IP_ADDR opline } zend_jit_set_last_valid_opline(opline); return 1; } static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) { if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } } if (!zend_jit_set_ip(Dst, opline)) { return 0; } reuse_ip = 0; return 1; } static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) { #if 0 if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 | jne ->interrupt_handler #else | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 if (exit_addr) { | jne &exit_addr } else if (last_valid_opline == opline) { || zend_jit_use_last_valid_opline(); | jne ->interrupt_handler } else { | jne >1 |.cold_code |1: | LOAD_IP_ADDR opline | jmp ->interrupt_handler |.code } #endif return 1; } static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) { if (timeout_exit_addr) { | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 | je =>loop_label | jmp &timeout_exit_addr } else { | jmp =>loop_label } return 1; } static int zend_jit_check_exception(dasm_State **Dst) { | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler return 1; } static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) { if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->exception_handler_undef return 1; } return zend_jit_check_exception(Dst); } static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) { zend_regset regset = ZEND_REGSET_SCRATCH; #if ZTS if (1) { #else if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) { #endif /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ if (parent) { int i; int parent_vars_count = parent->exit_info[exit_num].stack_size; zend_jit_trace_stack *parent_stack = parent->stack_map + parent->exit_info[exit_num].stack_offset; for (i = 0; i < parent_vars_count; i++) { if (STACK_REG(parent_stack, i) != ZREG_NONE) { if (STACK_REG(parent_stack, i) < ZREG_NUM) { ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) { ZEND_REGSET_EXCL(regset, ZREG_R0); } } } } } if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { ZEND_REGSET_EXCL(regset, ZREG_R0); } current_trace_num = trace_num; | // EG(jit_trace_num) = trace_num; if (regset == ZEND_REGSET_EMPTY) { | push r0 | MEM_OP2_1_ZTS mov, dword, executor_globals, jit_trace_num, trace_num, r0 | pop r0 } else { zend_reg tmp = ZEND_REGSET_FIRST(regset); | MEM_OP2_1_ZTS mov, dword, executor_globals, jit_trace_num, trace_num, Ra(tmp) (void)tmp; } return 1; } /* This taken from LuaJIT. Thanks to Mike Pall. */ static uint32_t _asm_x86_inslen(const uint8_t* p) { static const uint8_t map_op1[256] = { 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20, 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51, 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, #if defined(__x86_64__) || defined(_M_X64) 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14, #else 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, #endif 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51, 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51, #if defined(__x86_64__) || defined(_M_X64) 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, #else 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, #endif 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05, 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51, 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51, 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92 }; static const uint8_t map_op2[256] = { 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94, 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51, 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46, 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93, 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93, 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52 }; uint32_t result = 0; uint32_t prefixes = 0; uint32_t x = map_op1[*p]; for (;;) { switch (x >> 4) { case 0: return result + x + (prefixes & 4); case 1: prefixes |= x; x = map_op1[*++p]; result++; break; case 2: x = map_op2[*++p]; break; case 3: p++; goto mrm; case 4: result -= (prefixes & 2); /* fallthrough */ case 5: return result + (x & 15); case 6: /* Group 3. */ if (p[1] & 0x38) { x = 2; } else if ((prefixes & 2) && (x == 0x66)) { x = 4; } goto mrm; case 7: /* VEX c4/c5. */ #if !defined(__x86_64__) && !defined(_M_X64) if (p[1] < 0xc0) { x = 2; goto mrm; } #endif if (x == 0x70) { x = *++p & 0x1f; result++; if (x >= 2) { p += 2; result += 2; goto mrm; } } p++; result++; x = map_op2[*++p]; break; case 8: result -= (prefixes & 2); /* fallthrough */ case 9: mrm: /* ModR/M and possibly SIB. */ result += (x & 15); x = *++p; switch (x >> 6) { case 0: if ((x & 7) == 5) { return result + 4; } break; case 1: result++; break; case 2: result += 4; break; case 3: return result; } if ((x & 7) == 4) { result++; if (x < 0x40 && (p[1] & 7) == 5) { result += 4; } } return result; } } } typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) { int ret = 0; uint8_t *p, *end; if (jmp_table_size) { const void **jmp_slot = (const void **)((char*)code + size); size -= jmp_table_size * sizeof(void*); do { jmp_slot--; if (*jmp_slot == from_addr) { *jmp_slot = to_addr; ret++; } } while (--jmp_table_size); } p = (uint8_t*)code; end = p + size - 5; while (p < end) { if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) { *(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6)); ret++; } else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) { *(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5)); ret++; } p += _asm_x86_inslen(p); } #ifdef HAVE_VALGRIND VALGRIND_DISCARD_TRANSLATIONS(code, size); #endif return ret; } static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) { return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); } static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) { const void *link_addr; size_t prologue_size; /* Skip prologue. */ // TODO: don't hardcode this ??? if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE prologue_size = 0; #elif defined(__x86_64__) || defined(_M_X64) // sub r4, HYBRID_SPAD prologue_size = 4; #else // sub r4, HYBRID_SPAD prologue_size = 3; #endif } else if (GCC_GLOBAL_REGS) { // sub r4, SPAD // stack alignment #if defined(__x86_64__) || defined(_M_X64) prologue_size = 4; #else prologue_size = 3; #endif } else { // sub r4, NR_SPAD // stack alignment // mov aword T2, FP // save FP // mov aword T3, RX // save IP // mov FP, FCARG1a #if defined(__x86_64__) || defined(_M_X64) prologue_size = 17; #else prologue_size = 12; #endif } link_addr = (const void*)((const char*)t->code_start + prologue_size); if (timeout_exit_addr) { /* Check timeout for links to LOOP */ | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 | je &link_addr | jmp &timeout_exit_addr } else { | jmp &link_addr } return 1; } static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) { #if 0 | jmp ->trace_escape #else if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD if (!original_handler) { | JMP_IP } else { | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] | jmp aword [IP + r0] } } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment if (!original_handler) { | JMP_IP } else { | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] | jmp aword [IP + r0] } } else { if (original_handler) { | mov FCARG1a, FP | mov r0, EX->func | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] | call aword [IP + r0] } | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 2 // ZEND_VM_LEAVE | ret } #endif return 1; } static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_Z_TYPE FP + var, type, &exit_addr return 1; } static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | GET_ZVAL_LVAL ZREG_FCARG1a, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var) if (op_info & MAY_BE_ARRAY_PACKED) { | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz &exit_addr } else { | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jnz &exit_addr } return 1; } static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { zend_jit_op_array_trace_extension *jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); size_t offset = jit_extension->offset; const void *handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } if (!GCC_GLOBAL_REGS) { | mov FCARG1a, FP } | EXT_CALL handler, r0 if (may_throw && opline->opcode != ZEND_RETURN && opline->opcode != ZEND_RETURN_BY_REF) { | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 | jne ->exception_handler } while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { trace++; } if (!GCC_GLOBAL_REGS && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { if (opline->opcode == ZEND_RETURN || opline->opcode == ZEND_RETURN_BY_REF || opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME || opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_GENERATOR_CREATE) { | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r1 } } if (zend_jit_trace_may_exit(op_array, opline)) { if (opline->opcode == ZEND_RETURN || opline->opcode == ZEND_RETURN_BY_REF || opline->opcode == ZEND_GENERATOR_CREATE) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { #if 0 /* this check should be handled by the following OPLINE guard or jmp [IP] */ | cmp IP, zend_jit_halt_op | je ->trace_halt #endif } else if (GCC_GLOBAL_REGS) { | test IP, IP | je ->trace_halt } else { | test eax, eax | jl ->trace_halt } } else if (opline->opcode == ZEND_EXIT || opline->opcode == ZEND_GENERATOR_RETURN || opline->opcode == ZEND_YIELD || opline->opcode == ZEND_YIELD_FROM) { | jmp ->trace_halt } if (trace->op != ZEND_JIT_TRACE_END || (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { const zend_op *next_opline = trace->opline; const zend_op *exit_opline = NULL; uint32_t exit_point; const void *exit_addr; uint32_t old_info = 0; uint32_t old_res_info = 0; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; if (zend_is_smart_branch(opline)) { bool exit_if_true = 0; exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); } else { switch (opline->opcode) { case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: case ZEND_JMP_SET: case ZEND_COALESCE: case ZEND_JMP_NULL: case ZEND_FE_RESET_R: case ZEND_FE_RESET_RW: exit_opline = (trace->opline == opline + 1) ? OP_JMP_ADDR(opline, opline->op2) : opline + 1; break; case ZEND_JMPZNZ: exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : OP_JMP_ADDR(opline, opline->op2); break; case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: if (opline->op2_type == IS_CV) { old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); } exit_opline = (trace->opline == opline + 1) ? ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : opline + 1; break; } } if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); } exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } switch (opline->opcode) { case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: if (opline->op2_type == IS_CV) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); } break; } if (!exit_addr) { return 0; } | CMP_IP next_opline | jne &exit_addr } } zend_jit_set_last_valid_opline(trace->opline); return 1; } static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) { const void *handler; if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { handler = zend_get_opcode_handler_func(opline); } else { handler = opline->handler; } if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } if (!GCC_GLOBAL_REGS) { | mov FCARG1a, FP } | EXT_CALL handler, r0 if (may_throw) { zend_jit_check_exception(Dst); } /* Skip the following OP_DATA */ switch (opline->opcode) { case ZEND_ASSIGN_DIM: case ZEND_ASSIGN_OBJ: case ZEND_ASSIGN_STATIC_PROP: case ZEND_ASSIGN_DIM_OP: case ZEND_ASSIGN_OBJ_OP: case ZEND_ASSIGN_STATIC_PROP_OP: case ZEND_ASSIGN_STATIC_PROP_REF: case ZEND_ASSIGN_OBJ_REF: zend_jit_set_last_valid_opline(opline + 2); break; default: zend_jit_set_last_valid_opline(opline + 1); break; } return 1; } static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) { if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { if (opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME || opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_RETURN) { /* Use inlined HYBRID VM handler */ const void *handler = opline->handler; | ADD_HYBRID_SPAD | EXT_JMP handler, r0 } else { const void *handler = zend_get_opcode_handler_func(opline); | EXT_CALL handler, r0 | ADD_HYBRID_SPAD | JMP_IP } } else { const void *handler = opline->handler; if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment } else { | mov FCARG1a, FP | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment } | EXT_JMP handler, r0 } zend_jit_reset_last_valid_opline(); return 1; } static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) { uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | CMP_IP opline | jne &exit_addr zend_jit_set_last_valid_opline(opline); return 1; } static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) { | jmp =>target_label return 1; } static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) { | CMP_IP next_opline | jne =>target_label zend_jit_set_last_valid_opline(next_opline); return 1; } #ifdef CONTEXT_THREADED_JIT static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) { if (!zend_jit_handler(Dst, opline, 1)) return 0; if (opline->opcode == ZEND_DO_UCALL) { | call ->context_threaded_call } else { const zend_op *next_opline = opline + 1; | CMP_IP next_opline | je =>next_block | call ->context_threaded_call } return 1; } #endif static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) { #ifdef CONTEXT_THREADED_JIT return zend_jit_context_threaded_call(Dst, opline, next_block); #else return zend_jit_tail_handler(Dst, opline); #endif } static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) { ZEND_ASSERT(Z_MODE(src) == IS_REG); ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) if (set_type) { | SET_ZVAL_TYPE_INFO dst, IS_LONG } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | SSE_SET_ZVAL_DVAL dst, Z_REG(src) if (set_type) { | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE } } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); ZEND_ASSERT(Z_MODE(dst) == IS_REG); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | GET_ZVAL_LVAL Z_REG(dst), src } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | SSE_GET_ZVAL_DVAL Z_REG(dst), src } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) { zend_jit_addr src = ZEND_ADDR_REG(reg); zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); return zend_jit_spill_store(Dst, src, dst, info, set_type); } static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) { if (Z_MODE(src) == IS_REG && Z_STORE(src)) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); return zend_jit_spill_store(Dst, src, dst, info, 1); } return 1; } static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) { if (Z_MODE(src) == IS_REG && Z_STORE(src)) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); bool set_type = 1; if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { set_type = 0; } } return zend_jit_spill_store(Dst, src, dst, info, set_type); } return 1; } static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) { zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); zend_jit_addr dst = ZEND_ADDR_REG(reg); return zend_jit_load_reg(Dst, src, dst, info); } static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { if (!zend_jit_same_addr(src, dst)) { if (Z_MODE(src) == IS_REG) { if (Z_MODE(dst) == IS_REG) { if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) } else { ZEND_UNREACHABLE(); } } else if (Z_MODE(dst) == IS_MEM_ZVAL) { if (!Z_LOAD(src) && !Z_STORE(src)) { if (!zend_jit_spill_store(Dst, src, dst, info, JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) )) { return 0; } } } else { ZEND_UNREACHABLE(); } } else if (Z_MODE(src) == IS_MEM_ZVAL) { if (Z_MODE(dst) == IS_REG) { if (!zend_jit_load_reg(Dst, src, dst, info)) { return 0; } } else { ZEND_UNREACHABLE(); } } else { ZEND_UNREACHABLE(); } } return 1; } static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { if (!zend_jit_save_call_chain(Dst, -1)) { return 0; } } ZEND_ASSERT(opline); | LOAD_IP_ADDR (opline - 1) | jmp ->trace_escape |1: return 1; } static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); if (reg == ZREG_LONG_MIN_MINUS_1) { |.if X64 | SET_ZVAL_LVAL dst, 0x00000000 | SET_ZVAL_W2 dst, 0xc3e00000 |.else | SET_ZVAL_LVAL dst, 0x00200000 | SET_ZVAL_W2 dst, 0xc1e00000 |.endif | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE } else if (reg == ZREG_LONG_MIN) { |.if X64 | SET_ZVAL_LVAL dst, 0x00000000 | SET_ZVAL_W2 dst, 0x80000000 |.else | SET_ZVAL_LVAL dst, ZEND_LONG_MIN |.endif | SET_ZVAL_TYPE_INFO dst, IS_LONG } else if (reg == ZREG_LONG_MAX) { |.if X64 | SET_ZVAL_LVAL dst, 0xffffffff | SET_ZVAL_W2 dst, 0x7fffffff |.else | SET_ZVAL_LVAL dst, ZEND_LONG_MAX |.endif | SET_ZVAL_TYPE_INFO dst, IS_LONG } else if (reg == ZREG_LONG_MAX_PLUS_1) { |.if X64 | SET_ZVAL_LVAL dst, 0 | SET_ZVAL_W2 dst, 0x43e00000 |.else | SET_ZVAL_LVAL dst, 0 | SET_ZVAL_W2 dst, 0x41e00000 |.endif | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE } else if (reg == ZREG_NULL) { | SET_ZVAL_TYPE_INFO dst, IS_NULL } else if (reg == ZREG_ZVAL_TRY_ADDREF) { | IF_NOT_ZVAL_REFCOUNTED dst, >1 | GET_ZVAL_PTR r1, dst | GC_ADDREF r1 |1: } else if (reg == ZREG_ZVAL_COPY_GPR0) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2 | TRY_ADDREF -1, ch, r2 } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_free_trampoline(dasm_State **Dst) { | /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE | jz >1 | mov FCARG1a, r0 | EXT_CALL zend_jit_free_trampoline_helper, r0 |1: return 1; } static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { return 0; } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1) } else { | LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1) } if (may_overflow && (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { int32_t exit_point; const void *exit_addr; zend_jit_trace_stack *stack; uint32_t old_op1_info, old_res_info = 0; stack = JIT_G(current_frame)->stack; old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); } else { SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); } if (opline->result_type != IS_UNUSED) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); if (opline->opcode == ZEND_PRE_INC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); } else if (opline->opcode == ZEND_PRE_DEC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); } else if (opline->opcode == ZEND_POST_INC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); } else if (opline->opcode == ZEND_POST_DEC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); } } exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); | jo &exit_addr if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); if (opline->result_type != IS_UNUSED) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } } else if (may_overflow) { | jo >1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } |.cold_code |1: if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { |.if X64 | mov64 rax, 0x43e0000000000000 | SET_ZVAL_LVAL op1_def_addr, rax |.else | SET_ZVAL_LVAL op1_def_addr, 0 | SET_ZVAL_W2 op1_def_addr, 0x41e00000 |.endif } else { |.if X64 | mov64 rax, 0xc3e0000000000000 | SET_ZVAL_LVAL op1_def_addr, rax |.else | SET_ZVAL_LVAL op1_def_addr, 0x00200000 | SET_ZVAL_W2 op1_def_addr, 0xc1e00000 |.endif } if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE } if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 } | jmp >3 |.code } else { if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | SET_EX_OPLINE opline, r0 if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | mov FCARG1d, opline->op1.var | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL | EXT_CALL zend_jit_undefined_op_helper, r0 op1_info |= MAY_BE_NULL; } |2: | LOAD_ZVAL_ADDR FCARG1a, op1_addr | // ZVAL_DEREF(var_ptr); if (op1_info & MAY_BE_REF) { | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 | GET_Z_PTR FCARG1a, FCARG1a | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 | jz >1 if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR FCARG2a, res_addr } else { | xor FCARG2a, FCARG2a } if (opline->opcode == ZEND_PRE_INC) { | EXT_CALL zend_jit_pre_inc_typed_ref, r0 } else if (opline->opcode == ZEND_PRE_DEC) { | EXT_CALL zend_jit_pre_dec_typed_ref, r0 } else if (opline->opcode == ZEND_POST_INC) { | EXT_CALL zend_jit_post_inc_typed_ref, r0 } else if (opline->opcode == ZEND_POST_DEC) { | EXT_CALL zend_jit_post_dec_typed_ref, r0 } else { ZEND_UNREACHABLE(); } zend_jit_check_exception(Dst); | jmp >3 |1: | lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)] |2: } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 | TRY_ADDREF op1_info, ah, r2 } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { | LOAD_ZVAL_ADDR FCARG2a, res_addr | EXT_CALL zend_jit_pre_inc, r0 } else { | EXT_CALL increment_function, r0 } } else { if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { | LOAD_ZVAL_ADDR FCARG2a, res_addr | EXT_CALL zend_jit_pre_dec, r0 } else { | EXT_CALL decrement_function, r0 } } if (may_throw) { zend_jit_check_exception(Dst); } } else { zend_reg tmp_reg; if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 } if (Z_MODE(op1_def_addr) == IS_REG) { tmp_reg = Z_REG(op1_def_addr); } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { tmp_reg = Z_REG(op1_addr); } else { tmp_reg = ZREG_XMM0; } | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { if (CAN_USE_AVX()) { | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] } else { | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] } } else { if (CAN_USE_AVX()) { | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] } else { | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] } } | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 | TRY_ADDREF op1_def_info, ah, r1 } } | jmp >3 |.code } |3: if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { return 0; } if (opline->result_type != IS_UNUSED) { if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } } return 1; } static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) { if ((opline+1)->opcode == ZEND_OP_DATA && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { return 1; } return ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); } static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend_uchar opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_overflow) { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; zend_reg tmp_reg = ZREG_R0; if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { if (may_overflow && (res_info & MAY_BE_GUARD) && JIT_G(current_frame) && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { result_reg = ZREG_R0; } else { result_reg = Z_REG(res_addr); } } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { result_reg = Z_REG(op1_addr); } else if (Z_REG(res_addr) != ZREG_R0) { result_reg = ZREG_R0; } else { /* ASSIGN_DIM_OP */ result_reg = ZREG_FCARG1a; tmp_reg = ZREG_FCARG1a; } if (opcode == ZEND_MUL && ((Z_MODE(op2_addr) == IS_CONST_ZVAL && IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) || (Z_MODE(op1_addr) == IS_CONST_ZVAL && IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { if (Z_MODE(op1_addr) == IS_REG && Z_LVAL_P(Z_ZV(op2_addr)) == 2) { | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] } else { | GET_ZVAL_LVAL result_reg, op1_addr | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } } else { if (Z_MODE(op2_addr) == IS_REG && Z_LVAL_P(Z_ZV(op1_addr)) == 2) { | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] } else { | GET_ZVAL_LVAL result_reg, op2_addr | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) } } } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { | GET_ZVAL_LVAL result_reg, op1_addr | shr Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op1_addr) == IS_REG && Z_MODE(op2_addr) == IS_CONST_ZVAL) { | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))] } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op2_addr) == IS_REG && Z_MODE(op1_addr) == IS_CONST_ZVAL) { | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))] } else if (opcode == ZEND_SUB && !may_overflow && Z_MODE(op1_addr) == IS_REG && Z_MODE(op2_addr) == IS_CONST_ZVAL) { | lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))] } else { | GET_ZVAL_LVAL result_reg, op1_addr if ((opcode == ZEND_ADD || opcode == ZEND_SUB) && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { /* +/- 0 */ may_overflow = 0; } else if (same_ops && opcode != ZEND_DIV) { | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) } else { | LONG_MATH opcode, result_reg, op2_addr } } if (may_overflow) { if (res_info & MAY_BE_GUARD) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { | jo &exit_addr if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { | mov Ra(Z_REG(res_addr)), Ra(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | jno &exit_addr } else { ZEND_UNREACHABLE(); } } else { if (res_info & MAY_BE_LONG) { | jo >1 } else { | jno >1 } } } if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { | SET_ZVAL_LVAL res_addr, Ra(result_reg) if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } } } if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { zend_reg tmp_reg1 = ZREG_XMM0; zend_reg tmp_reg2 = ZREG_XMM1; if (res_info & MAY_BE_LONG) { |.cold_code |1: } do { if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) && ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) { if (opcode == ZEND_ADD) { |.if X64 | mov64 rax, 0x43e0000000000000 if (Z_MODE(res_addr) == IS_REG) { | movd xmm(Z_REG(res_addr)-ZREG_XMM0), rax } else { | SET_ZVAL_LVAL res_addr, rax } |.else | SET_ZVAL_LVAL res_addr, 0 | SET_ZVAL_W2 res_addr, 0x41e00000 |.endif break; } else if (opcode == ZEND_SUB) { |.if X64 | mov64 rax, 0xc3e0000000000000 if (Z_MODE(res_addr) == IS_REG) { | movd xmm(Z_REG(res_addr)-ZREG_XMM0), rax } else { | SET_ZVAL_LVAL res_addr, rax } |.else | SET_ZVAL_LVAL res_addr, 0x00200000 | SET_ZVAL_W2 res_addr, 0xc1e00000 |.endif break; } } | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg if (CAN_USE_AVX()) { | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 } else { | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 } | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 } while (0); if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } if (res_info & MAY_BE_LONG) { | jmp >2 |.code } |2: } return 1; } static int zend_jit_math_long_double(dasm_State **Dst, zend_uchar opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { zend_reg result_reg = (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; zend_reg tmp_reg; if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { /* ASSIGN_DIM_OP */ tmp_reg = ZREG_R1; } else { tmp_reg = ZREG_R0; } | SSE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { /* ASSIGN_DIM_OP */ if (CAN_USE_AVX()) { | AVX_MATH opcode, result_reg, result_reg, op2_addr, r1 } else { | SSE_MATH opcode, result_reg, op2_addr, r1 } } else { if (CAN_USE_AVX()) { | AVX_MATH opcode, result_reg, result_reg, op2_addr, r0 } else { | SSE_MATH opcode, result_reg, op2_addr, r0 } } | SSE_SET_ZVAL_DVAL res_addr, result_reg if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } } return 1; } static int zend_jit_math_double_long(dasm_State **Dst, zend_uchar opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { zend_reg result_reg, tmp_reg_gp; if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { /* ASSIGN_DIM_OP */ tmp_reg_gp = ZREG_R1; } else { tmp_reg_gp = ZREG_R0; } if (zend_is_commutative(opcode) && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); } else { result_reg = ZREG_XMM0; } | SSE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { /* ASSIGN_DIM_OP */ if (CAN_USE_AVX()) { | AVX_MATH opcode, result_reg, result_reg, op1_addr, r1 } else { | SSE_MATH opcode, result_reg, op1_addr, r1 } } else { if (CAN_USE_AVX()) { | AVX_MATH opcode, result_reg, result_reg, op1_addr, r0 } else { | SSE_MATH opcode, result_reg, op1_addr, r0 } } } else { zend_reg tmp_reg; if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0; } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { result_reg = Z_REG(op1_addr); tmp_reg = ZREG_XMM0; } else { result_reg = ZREG_XMM0; tmp_reg = ZREG_XMM1; } if (CAN_USE_AVX()) { zend_reg op1_reg; if (Z_MODE(op1_addr) == IS_REG) { op1_reg = Z_REG(op1_addr); } else { | SSE_GET_ZVAL_DVAL result_reg, op1_addr op1_reg = result_reg; } if ((opcode == ZEND_ADD || opcode == ZEND_SUB) && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { /* +/- 0 */ } else { | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg } } else { | SSE_GET_ZVAL_DVAL result_reg, op1_addr if ((opcode == ZEND_ADD || opcode == ZEND_SUB) && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { /* +/- 0 */ } else { | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp | SSE_MATH_REG opcode, result_reg, tmp_reg } } } | SSE_SET_ZVAL_DVAL res_addr, result_reg if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } } } return 1; } static int zend_jit_math_double_double(dasm_State **Dst, zend_uchar opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { result_reg = Z_REG(op1_addr); } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { result_reg = Z_REG(op2_addr); } else { result_reg = ZREG_XMM0; } if (CAN_USE_AVX()) { zend_reg op1_reg; zend_jit_addr val_addr; if (Z_MODE(op1_addr) == IS_REG) { op1_reg = Z_REG(op1_addr); val_addr = op2_addr; } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { op1_reg = Z_REG(op2_addr); val_addr = op1_addr; } else { | SSE_GET_ZVAL_DVAL result_reg, op1_addr op1_reg = result_reg; val_addr = op2_addr; } if ((opcode == ZEND_MUL) && Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { /* ASSIGN_DIM_OP */ | AVX_MATH opcode, result_reg, op1_reg, val_addr, r1 } else { | AVX_MATH opcode, result_reg, op1_reg, val_addr, r0 } } else { zend_jit_addr val_addr; if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { | SSE_GET_ZVAL_DVAL result_reg, op2_addr val_addr = op1_addr; } else { | SSE_GET_ZVAL_DVAL result_reg, op1_addr val_addr = op2_addr; } if (same_ops) { | SSE_MATH_REG opcode, result_reg, result_reg } else if ((opcode == ZEND_MUL) && Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { | SSE_MATH_REG ZEND_ADD, result_reg, result_reg } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { /* ASSIGN_DIM_OP */ | SSE_MATH opcode, result_reg, val_addr, r1 } else { | SSE_MATH opcode, result_reg, val_addr, r0 } } | SSE_SET_ZVAL_DVAL res_addr, result_reg if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE } } } return 1; } static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_uchar opcode, zend_uchar op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, zend_uchar op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_overflow, int may_throw) /* Labels: 1,2,3,4,5,6 */ { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 |.cold_code |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | jmp >5 |.code } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } } if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { |.cold_code |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 } else { | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | jmp >5 } if (!same_ops) { |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | jmp >5 } |.code } } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } } if (!same_ops && (op2_info & MAY_BE_LONG)) { if (op2_info & MAY_BE_DOUBLE) { |.cold_code } |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { | jmp >5 |.code } } } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } } if (!same_ops && (op1_info & MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { |.cold_code } |1: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 } if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { | jmp >5 |.code } } } |5: if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { |.cold_code } |6: if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); | LOAD_ZVAL_ADDR FCARG1a, real_addr } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, res_addr } if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2a, op1_addr if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; } |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif | SET_EX_OPLINE opline, r0 if (opcode == ZEND_ADD) { | EXT_CALL add_function, r0 } else if (opcode == ZEND_SUB) { | EXT_CALL sub_function, r0 } else if (opcode == ZEND_MUL) { | EXT_CALL mul_function, r0 } else if (opcode == ZEND_DIV) { | EXT_CALL div_function, r0 } else { ZEND_UNREACHABLE(); } |.if not(X64) | add r4, 12 |.endif | FREE_OP op1_type, op1, op1_info, 0, opline | FREE_OP op2_type, op2, op2_info, 0, opline if (may_throw) { zend_jit_check_exception(Dst); } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { return 0; } } if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { | jmp <5 |.code } } return 1; } static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) { ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { return 0; } if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } return 1; } static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr) { zend_jit_addr op1_addr = OP1_ADDR(); zend_jit_addr op2_addr = OP2_ADDR(); | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr | EXT_CALL zend_jit_add_arrays_helper, r0 | SET_ZVAL_PTR res_addr, r0 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline return 1; } static int zend_jit_long_math_helper(dasm_State **Dst, const zend_op *opline, zend_uchar opcode, zend_uchar op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, zend_ssa_range *op1_range, zend_uchar op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, zend_ssa_range *op2_range, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_throw) /* Labels: 6 */ { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; zval tmp; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 } if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 } if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL && op1_range && op1_range->min >= 0) { zend_long l = Z_LVAL_P(Z_ZV(op2_addr)); if (zend_long_is_power_of_two(l)) { /* Optimisation for mod of power of 2 */ opcode = ZEND_BW_AND; ZVAL_LONG(&tmp, l - 1); op2_addr = ZEND_ADDR_CONST_ZVAL(&tmp); } } if (opcode == ZEND_MOD) { result_reg = ZREG_RAX; } else if (Z_MODE(res_addr) == IS_REG) { if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) && opline->op2_type != IS_CONST) { result_reg = ZREG_R0; } else { result_reg = Z_REG(res_addr); } } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { result_reg = Z_REG(op1_addr); } else if (Z_REG(res_addr) != ZREG_R0) { result_reg = ZREG_R0; } else { /* ASSIGN_DIM_OP */ if (sizeof(void*) == 4 && (opcode == ZEND_SL || opcode == ZEND_SR) && Z_MODE(op2_addr) != IS_CONST_ZVAL) { result_reg = ZREG_R2; } else { result_reg = ZREG_FCARG1a; } } if (opcode == ZEND_SL) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { if (EXPECTED(op2_lval > 0)) { | xor Ra(result_reg), Ra(result_reg) } else { | SET_EX_OPLINE opline, r0 | jmp ->negative_shift } } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] } else { | GET_ZVAL_LVAL result_reg, op1_addr | shl Ra(result_reg), op2_lval } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { | GET_ZVAL_LVAL ZREG_RCX, op2_addr } if (!op2_range || op2_range->min < 0 || op2_range->max >= SIZEOF_ZEND_LONG * 8) { | cmp r1, (SIZEOF_ZEND_LONG*8) | jae >1 |.cold_code |1: | cmp r1, 0 | mov Ra(result_reg), 0 | jg >1 | SET_EX_OPLINE opline, r0 | jmp ->negative_shift |.code } | GET_ZVAL_LVAL result_reg, op1_addr | shl Ra(result_reg), cl |1: } } else if (opcode == ZEND_SR) { | GET_ZVAL_LVAL result_reg, op1_addr if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { if (EXPECTED(op2_lval > 0)) { | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 } else { | SET_EX_OPLINE opline, r0 | jmp ->negative_shift } } else { | sar Ra(result_reg), op2_lval } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { | GET_ZVAL_LVAL ZREG_RCX, op2_addr } if (!op2_range || op2_range->min < 0 || op2_range->max >= SIZEOF_ZEND_LONG * 8) { | cmp r1, (SIZEOF_ZEND_LONG*8) | jae >1 |.cold_code |1: | cmp r1, 0 | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 | jg >1 | SET_EX_OPLINE opline, r0 | jmp ->negative_shift |.code } |1: | sar Ra(result_reg), cl } } else if (opcode == ZEND_MOD) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (op2_lval == 0) { | SET_EX_OPLINE opline, r0 | jmp ->mod_by_zero } else if (op2_lval == -1) { | xor Ra(result_reg), Ra(result_reg) } else { result_reg = ZREG_RDX; | GET_ZVAL_LVAL ZREG_RAX, op1_addr | GET_ZVAL_LVAL ZREG_RCX, op2_addr |.if X64 | cqo |.else | cdq |.endif | idiv Ra(ZREG_RCX) } } else { if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 } else if (Z_MODE(op2_addr) == IS_REG) { | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) } | jz >1 |.cold_code |1: | SET_EX_OPLINE opline, r0 | jmp ->mod_by_zero |.code } /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1 } else if (Z_MODE(op2_addr) == IS_REG) { | cmp Ra(Z_REG(op2_addr)), -1 } | jz >1 |.cold_code |1: | SET_ZVAL_LVAL res_addr, 0 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } } } | jmp >5 |.code } result_reg = ZREG_RDX; | GET_ZVAL_LVAL ZREG_RAX, op1_addr |.if X64 | cqo |.else | cdq |.endif if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] } else if (Z_MODE(op2_addr) == IS_REG) { | idiv Ra(Z_REG(op2_addr)) } } } else if (same_ops) { | GET_ZVAL_LVAL result_reg, op1_addr | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) } else { | GET_ZVAL_LVAL result_reg, op1_addr | LONG_MATH opcode, result_reg, op2_addr } if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { | SET_ZVAL_LVAL res_addr, Ra(result_reg) } if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } } } if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { |.cold_code } |6: if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); | LOAD_ZVAL_ADDR FCARG1a, real_addr } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, res_addr } if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2a, op1_addr if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; } |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif | SET_EX_OPLINE opline, r0 if (opcode == ZEND_BW_OR) { | EXT_CALL bitwise_or_function, r0 } else if (opcode == ZEND_BW_AND) { | EXT_CALL bitwise_and_function, r0 } else if (opcode == ZEND_BW_XOR) { | EXT_CALL bitwise_xor_function, r0 } else if (opcode == ZEND_SL) { | EXT_CALL shift_left_function, r0 } else if (opcode == ZEND_SR) { | EXT_CALL shift_right_function, r0 } else if (opcode == ZEND_MOD) { | EXT_CALL mod_function, r0 } else { ZEND_UNREACHABLE(); } |.if not(X64) | add r4, 12 |.endif | FREE_OP op1_type, op1, op1_info, 0, opline | FREE_OP op2_type, op2, op2_info, 0, opline if (may_throw) { zend_jit_check_exception(Dst); } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { return 0; } } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { | jmp >5 |.code } } |5: return 1; } static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) { ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, opline->result.var, res_addr, res_info, res_use_info, may_throw)) { return 0; } if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } return 1; } static int zend_jit_concat_helper(dasm_State **Dst, const zend_op *opline, zend_uchar op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, zend_uchar op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { #if 1 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 } if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 } if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, res_addr } | LOAD_ZVAL_ADDR FCARG2a, op2_addr | EXT_CALL zend_jit_fast_assign_concat_helper, r0 } else { if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, res_addr } | LOAD_ZVAL_ADDR FCARG2a, op1_addr |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif | EXT_CALL zend_jit_fast_concat_helper, r0 |.if not(X64) | add r4, 12 |.endif } /* concatination with empty string may increase refcount */ op1_info |= MAY_BE_RCN; op2_info |= MAY_BE_RCN; | FREE_OP op1_type, op1, op1_info, 0, opline | FREE_OP op2_type, op2, op2_info, 0, opline |5: } if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { |.cold_code |6: } #endif if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, res_addr } | LOAD_ZVAL_ADDR FCARG2a, op1_addr |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif | SET_EX_OPLINE opline, r0 | EXT_CALL concat_function, r0 |.if not(X64) | add r4, 12 |.endif /* concatination with empty string may increase refcount */ op1_info |= MAY_BE_RCN; op2_info |= MAY_BE_RCN; | FREE_OP op1_type, op1, op1_info, 0, opline | FREE_OP op2_type, op2, op2_info, 0, opline if (may_throw) { zend_jit_check_exception(Dst); } #if 1 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { | jmp <5 |.code } } #endif return 1; } static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { zend_jit_addr op1_addr, op2_addr; ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); op1_addr = OP1_ADDR(); op2_addr = OP2_ADDR(); return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); } static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) /* Labels: 1,2,3,4,5 */ { zend_jit_addr op2_addr = OP2_ADDR(); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R && !exit_addr) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if (op2_info & MAY_BE_LONG) { bool op2_loaded = 0; bool packed_loaded = 0; if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 } if (op1_info & MAY_BE_PACKED_GUARD) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } if (op1_info & MAY_BE_ARRAY_PACKED) { | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz &exit_addr } else { | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jnz &exit_addr } } if (type == BP_VAR_W) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr op2_loaded = 1; } if (op1_info & MAY_BE_ARRAY_PACKED) { zend_long val = -1; if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { val = Z_LVAL_P(Z_ZV(op2_addr)); if (val >= 0 && val < HT_MAX_SIZE) { packed_loaded = 1; } } else { if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr op2_loaded = 1; } packed_loaded = 1; } if (packed_loaded) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); if (op1_info & MAY_BE_ARRAY_HASH) { | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED | jz >4 // HASH_FIND } | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) |.if X64 | mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)] if (val == 0) { | test r0, r0 } else if (val > 0 && !op2_loaded) { | cmp r0, val } else { | cmp r0, FCARG2a } |.else if (val >= 0 && !op2_loaded) { | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val } else { | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a } |.endif if (type == BP_JIT_IS) { if (not_found_exit_addr) { | jbe ¬_found_exit_addr } else { | jbe >9 // NOT_FOUND } } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | jbe &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { | jbe ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | jbe >7 // NOT_FOUND } else { | jbe >2 // NOT_FOUND } | // _ret = &_ht->arData[_h].val; if (val >= 0) { | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] if (val != 0) { | add r0, val * sizeof(Bucket) } } else { |.if X64 | mov r0, FCARG2a | shl r0, 5 |.else | imul r0, FCARG2a, sizeof(Bucket) |.endif | add r0, aword [FCARG1a + offsetof(zend_array, arData)] } } } switch (type) { case BP_JIT_IS: if (op1_info & MAY_BE_ARRAY_HASH) { if (packed_loaded) { | jmp >5 } |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr } | EXT_CALL _zend_hash_index_find, r0 | test r0, r0 if (not_found_exit_addr) { | jz ¬_found_exit_addr } else { | jz >9 // NOT_FOUND } if (op2_info & MAY_BE_STRING) { | jmp >5 } } else if (packed_loaded) { if (op2_info & MAY_BE_STRING) { | jmp >5 } } else if (not_found_exit_addr) { | jmp ¬_found_exit_addr } else { | jmp >9 // NOT_FOUND } break; case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: if (packed_loaded) { if (op1_info & MAY_BE_ARRAY_HASH) { | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) { | IF_Z_TYPE r0, IS_UNDEF, &exit_addr } } else if (type == BP_VAR_IS && not_found_exit_addr) { | IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND } else { | IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND } } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | jmp &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { | jmp ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | jmp >7 // NOT_FOUND } else { | jmp >2 // NOT_FOUND } } if (op1_info & MAY_BE_ARRAY_HASH) { |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr } | EXT_CALL _zend_hash_index_find, r0 | test r0, r0 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | jz &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { | jz ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | jz >7 // NOT_FOUND } else { | jz >2 // NOT_FOUND } } |.cold_code |2: switch (type) { case BP_VAR_R: if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); | // retval = &EG(uninitialized_zval); | UNDEFINED_OFFSET opline | jmp >9 } break; case BP_VAR_IS: case BP_VAR_UNSET: if (!not_found_exit_addr && !found_exit_addr) { | // retval = &EG(uninitialized_zval); | SET_ZVAL_TYPE_INFO res_addr, IS_NULL | jmp >9 } break; default: ZEND_UNREACHABLE(); } |.code break; case BP_VAR_RW: if (packed_loaded) { | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 } |2: |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr } | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_hash_index_lookup_rw, r0 | test r0, r0 | jz >9 break; case BP_VAR_W: if (packed_loaded) { | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || packed_loaded) { |2: | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr } |.if X64 | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval |.else | sub r4, 12 | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 |.endif | EXT_CALL zend_hash_index_add_new, r0 |.if not(X64) | add r4, 12 |.endif if (op1_info & MAY_BE_ARRAY_HASH) { | jmp >8 } } if (op1_info & MAY_BE_ARRAY_HASH) { |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr } | EXT_CALL zend_hash_index_lookup, r0 } break; default: ZEND_UNREACHABLE(); } if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { | jmp >8 } } if (op2_info & MAY_BE_STRING) { |3: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 } | // offset_key = Z_STR_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr | // retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: if (opline->op2_type != IS_CONST) { | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' | jle >1 |.cold_code |1: | EXT_CALL zend_jit_symtable_find, r0 | jmp >1 |.code | EXT_CALL zend_hash_find, r0 |1: } else { | EXT_CALL _zend_hash_find_known_hash, r0 } | test r0, r0 if (not_found_exit_addr) { | jz ¬_found_exit_addr } else { | jz >9 // NOT_FOUND } break; case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: if (opline->op2_type != IS_CONST) { | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' | jle >1 |.cold_code |1: | EXT_CALL zend_jit_symtable_find, r0 | jmp >1 |.code | EXT_CALL zend_hash_find, r0 |1: } else { | EXT_CALL _zend_hash_find_known_hash, r0 } | test r0, r0 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | jz &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { | jz ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | jz >7 // NOT_FOUND } else { | jz >2 // NOT_FOUND |.cold_code |2: switch (type) { case BP_VAR_R: // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); | UNDEFINED_INDEX opline | jmp >9 break; case BP_VAR_IS: case BP_VAR_UNSET: | // retval = &EG(uninitialized_zval); | SET_ZVAL_TYPE_INFO res_addr, IS_NULL | jmp >9 break; default: ZEND_UNREACHABLE(); } |.code } break; case BP_VAR_RW: | SET_EX_OPLINE opline, r0 if (opline->op2_type != IS_CONST) { | EXT_CALL zend_jit_symtable_lookup_rw, r0 } else { | EXT_CALL zend_jit_hash_lookup_rw, r0 } | test r0, r0 | jz >9 break; case BP_VAR_W: if (opline->op2_type != IS_CONST) { | EXT_CALL zend_jit_symtable_lookup_w, r0 } else { | EXT_CALL zend_hash_lookup, r0 } break; default: ZEND_UNREACHABLE(); } } if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { |5: if (op1_info & MAY_BE_ARRAY_OF_REF) { | ZVAL_DEREF r0, MAY_BE_REF } | cmp byte [r0 + 8], IS_NULL if (not_found_exit_addr) { | jle ¬_found_exit_addr } else if (found_exit_addr) { | jg &found_exit_addr } else { | jle >9 // NOT FOUND } } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { |.cold_code |3: } | SET_EX_OPLINE opline, r0 | LOAD_ZVAL_ADDR FCARG2a, op2_addr switch (type) { case BP_VAR_R: |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_fetch_dim_r_helper, r0 |.if not(X64) | add r4, 12 |.endif | jmp >9 break; case BP_JIT_IS: | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 | test r0, r0 if (not_found_exit_addr) { | je ¬_found_exit_addr if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { | jmp >8 } } else if (found_exit_addr) { | jne &found_exit_addr if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { | jmp >9 } } else { | jne >8 | jmp >9 } break; case BP_VAR_IS: case BP_VAR_UNSET: |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_fetch_dim_is_helper, r0 |.if not(X64) | add r4, 12 |.endif | jmp >9 break; case BP_VAR_RW: | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 | test r0, r0 | jne >8 | jmp >9 break; case BP_VAR_W: | EXT_CALL zend_jit_fetch_dim_w_helper, r0 | test r0, r0 | jne >8 | jmp >9 break; default: ZEND_UNREACHABLE(); } if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { |.code } } return 1; } static int zend_jit_simple_assign(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr, uint32_t var_info, uint32_t var_def_info, zend_uchar val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr, int in_cold, int save_r1) /* Labels: 1,2,3 */ { zend_reg tmp_reg; if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) { tmp_reg = ZREG_R0; } else { /* ASSIGN_DIM */ tmp_reg = ZREG_FCARG1a; } if (Z_MODE(val_addr) == IS_CONST_ZVAL) { zval *zv = Z_ZV(val_addr); if (!res_addr) { | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg } else { | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg } if (Z_REFCOUNTED_P(zv)) { if (!res_addr) { | ADDREF_CONST zv, Ra(tmp_reg) } else { | ADDREF_CONST_2 zv, Ra(tmp_reg) } } } else { if (val_info & MAY_BE_UNDEF) { if (in_cold) { | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 } else { | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 |.cold_code |1: } | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); if (save_r1) { | mov aword T1, FCARG1a // save } | SET_ZVAL_TYPE_INFO var_addr, IS_NULL if (res_addr) { | SET_ZVAL_TYPE_INFO res_addr, IS_NULL } if (opline) { | SET_EX_OPLINE opline, Ra(tmp_reg) } ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); | mov FCARG1d, Z_OFFSET(val_addr) | EXT_CALL zend_jit_undefined_op_helper, r0 if (save_r1) { | mov FCARG1a, aword T1 // restore } | jmp >3 if (in_cold) { |2: } else { |.code } } if (val_info & MAY_BE_REF) { if (val_type == IS_CV) { ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR r2, val_addr } | ZVAL_DEREF r2, val_info val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); } else { zend_jit_addr ref_addr; if (in_cold) { | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 } else { | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 |.cold_code |1: } | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR r2, val_addr | GC_DELREF r2 | // ZVAL_COPY_VALUE(return_value, &ref->value); ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); if (!res_addr) { | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_R2, tmp_reg } else { | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_R2, tmp_reg } | je >2 | IF_NOT_REFCOUNTED dh, >3 if (!res_addr) { | GC_ADDREF Ra(tmp_reg) } else { | add dword [Ra(tmp_reg)], 2 } | jmp >3 |2: if (res_addr) { | IF_NOT_REFCOUNTED dh, >2 | GC_ADDREF Ra(tmp_reg) |2: } if (save_r1) { | mov aword T1, FCARG1a // save } | EFREE_REFERENCE aword [Ra(Z_REG(val_addr))+Z_OFFSET(val_addr)] if (save_r1) { | mov FCARG1a, aword T1 // restore } | jmp >3 if (in_cold) { |1: } else { |.code } } } if (!res_addr) { | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg } else { | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg } if (val_type == IS_CV) { if (!res_addr) { | TRY_ADDREF val_info, dh, Ra(tmp_reg) } else { | TRY_ADDREF_2 val_info, dh, Ra(tmp_reg) } } else { if (res_addr) { | TRY_ADDREF val_info, dh, Ra(tmp_reg) } } |3: } return 1; } static int zend_jit_assign_to_typed_ref(dasm_State **Dst, const zend_op *opline, zend_uchar val_type, zend_jit_addr val_addr, bool check_exception) { | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 | jnz >2 |.cold_code |2: if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2a, val_addr } if (opline) { | SET_EX_OPLINE opline, r0 } if (val_type == IS_CONST) { | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 } else if (val_type == IS_TMP_VAR) { | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 } else if (val_type == IS_VAR) { | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 } else if (val_type == IS_CV) { | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 } else { ZEND_UNREACHABLE(); } if (check_exception) { | // if (UNEXPECTED(EG(exception) != NULL)) { | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | je >8 // END OF zend_jit_assign_to_variable() | jmp ->exception_handler } else { | jmp >8 } |.code return 1; } static int zend_jit_assign_to_variable_call(dasm_State **Dst, const zend_op *opline, zend_jit_addr __var_use_addr, zend_jit_addr var_addr, uint32_t __var_info, uint32_t __var_def_info, zend_uchar val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr __res_addr, bool __check_exception) { if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, var_addr } if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2a, val_addr } if (opline) { | SET_EX_OPLINE opline, r0 } if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | call ->assign_tmp } else if (val_type == IS_CONST) { | call ->assign_const } else if (val_type == IS_TMP_VAR) { | call ->assign_tmp } else if (val_type == IS_VAR) { if (!(val_info & MAY_BE_REF)) { | call ->assign_tmp } else { | call ->assign_var } } else if (val_type == IS_CV) { if (!(val_info & MAY_BE_REF)) { | call ->assign_cv_noref } else { | call ->assign_cv } } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_assign_to_variable(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_use_addr, zend_jit_addr var_addr, uint32_t var_info, uint32_t var_def_info, zend_uchar val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr, bool check_exception) /* Labels: 1,2,3,4,5,8 */ { int done = 0; zend_reg ref_reg, tmp_reg; if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) { ref_reg = ZREG_FCARG1a; tmp_reg = ZREG_R0; } else { /* ASSIGN_DIM */ ref_reg = ZREG_R0; tmp_reg = ZREG_FCARG1a; } if (var_info & MAY_BE_REF) { if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { | LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); } | // if (Z_ISREF_P(variable_ptr)) { | IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >1 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { | GET_Z_PTR FCARG1a, Ra(ref_reg) if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, check_exception)) { return 0; } | lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)] |1: } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { if (RC_MAY_BE_1(var_info)) { int in_cold = 0; if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_ZVAL_REFCOUNTED var_use_addr, >1 |.cold_code |1: in_cold = 1; } if (Z_REG(var_use_addr) == ZREG_FCARG1a || Z_REG(var_use_addr) == ZREG_R0) { bool keep_gc = 0; | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr if (tmp_reg == ZREG_FCARG1a) { if (Z_MODE(val_addr) == IS_REG) { keep_gc = 1; } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { keep_gc = 1; } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { if (sizeof(void*) == 4) { keep_gc = 1; } else { zval *zv = Z_ZV(val_addr); if (Z_TYPE_P(zv) == IS_DOUBLE) { if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) { keep_gc = 1; } } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { keep_gc = 1; } } } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { keep_gc = 1; } } } if (!keep_gc) { | mov aword T1, Ra(tmp_reg) // save } if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) { return 0; } if (!keep_gc) { | mov FCARG1a, aword T1 // restore } } else { | GET_ZVAL_PTR FCARG1a, var_use_addr if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { return 0; } } | GC_DELREF FCARG1a if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { | jnz >4 } else { | jnz >8 } | ZVAL_DTOR_FUNC var_info, opline if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { if (check_exception) { | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | je >8 | jmp ->exception_handler } else { | jmp >8 } } if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { |4: | IF_GC_MAY_NOT_LEAK FCARG1a, >8 | EXT_CALL gc_possible_root, r0 if (in_cold) { | jmp >8 } } if (in_cold) { |.code } else { done = 1; } } else /* if (RC_MAY_BE_N(var_info)) */ { if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5 } if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { if (Z_REG(var_use_addr) == ZREG_FP) { | mov T1, Ra(Z_REG(var_use_addr)) // save } | GET_ZVAL_PTR FCARG1a, var_use_addr | GC_DELREF FCARG1a | IF_GC_MAY_NOT_LEAK FCARG1a, >5 | EXT_CALL gc_possible_root, r0 if (Z_REG(var_use_addr) != ZREG_FP) { | mov Ra(Z_REG(var_use_addr)), T1 // restore } } else { | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr | GC_DELREF Ra(tmp_reg) } |5: } } if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) { return 0; } |8: return 1; } static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, int may_throw) { zend_jit_addr op2_addr, op3_addr, res_addr; op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; op3_addr = OP1_DATA_ADDR(); if (opline->result_type == IS_UNUSED) { res_addr = 0; } else { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr val_info &= ~MAY_BE_UNDEF; } if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 | GET_Z_PTR FCARG2a, FCARG1a | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] | jmp >3 |.cold_code |2: | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 | test r0, r0 | mov FCARG1a, r0 | jne >1 | jmp ->exception_handler_undef |.code |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >7 } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | mov T1, Ra(Z_REG(op1_addr)) // save } | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX | mov FCARG1a, r0 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |6: if (opline->op2_type == IS_UNUSED) { uint32_t var_info = MAY_BE_NULL; zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | EXT_CALL zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 | jz >1 |.cold_code |1: | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); | CANNOT_ADD_ELEMENT opline | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | jmp >9 |.code if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { return 0; } } else { uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, NULL, NULL, NULL)) { return 0; } if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { var_info |= MAY_BE_REF; } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { var_info |= MAY_BE_RC1; } |8: | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); if (opline->op1_type == IS_VAR) { ZEND_ASSERT(opline->result_type == IS_UNUSED); if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { return 0; } } else { if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { return 0; } } } } if (((op1_info & MAY_BE_ARRAY) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |.cold_code |7: } if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && (op1_info & MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >2 } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | mov T1, Ra(Z_REG(op1_addr)) // save } | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX | mov FCARG1a, r0 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | jmp <6 |2: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | SET_EX_OPLINE opline, r0 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_UNUSED) { | xor FCARG2a, FCARG2a } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } |.if not(X64) | sub r4, 8 |.endif if (opline->result_type == IS_UNUSED) { |.if X64 | xor CARG4, CARG4 |.else | push 0 |.endif } else { |.if X64 | LOAD_ZVAL_ADDR CARG4, res_addr |.else | PUSH_ZVAL_ADDR res_addr, r0 |.endif } |.if X64 | LOAD_ZVAL_ADDR CARG3, op3_addr |.else | PUSH_ZVAL_ADDR op3_addr, r0 |.endif | EXT_CALL zend_jit_assign_dim_helper, r0 |.if not(X64) | add r4, 8 |.endif #ifdef ZEND_JIT_USE_RC_INFERENCE if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { /* ASSIGN_DIM may increase refcount of the value */ val_info |= MAY_BE_RCN; } #endif | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | jmp >9 // END } |.code } } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { /* ASSIGN_DIM may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif |9: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline if (may_throw) { zend_jit_check_exception(Dst); } return 1; } static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, int may_throw) { zend_jit_addr op2_addr, op3_addr, var_addr; ZEND_ASSERT(opline->result_type == IS_UNUSED); op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; op3_addr = OP1_DATA_ADDR(); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 | GET_Z_PTR FCARG2a, FCARG1a | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] | jmp >3 |.cold_code |2: | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 | test r0, r0 | mov FCARG1a, r0 | jne >1 | jmp ->exception_handler_undef |.code |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >7 } if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 } | SET_EX_OPLINE opline, r0 | mov FCARG1a, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | mov T1, Ra(Z_REG(op1_addr)) // save } | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX | mov FCARG1a, r0 if (op1_info & MAY_BE_ARRAY) { | jmp >1 |.code |1: } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { uint32_t var_info; uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); |6: if (opline->op2_type == IS_UNUSED) { var_info = MAY_BE_NULL; | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | EXT_CALL zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 | jz >1 |.cold_code |1: | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); | CANNOT_ADD_ELEMENT opline | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | jmp >9 |.code } else { var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { var_info |= MAY_BE_REF; } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { var_info |= MAY_BE_RC1; } if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, NULL, NULL, NULL)) { return 0; } |8: if (op1_info & (MAY_BE_ARRAY_OF_REF)) { binary_op_type binary_op = get_binary_op(opline->extended_value); | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 | GET_Z_PTR FCARG1a, r0 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 | jnz >2 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] |.cold_code |2: | LOAD_ZVAL_ADDR FCARG2a, op3_addr |.if X64 | LOAD_ADDR CARG3, binary_op |.else | sub r4, 12 | PUSH_ADDR binary_op, r0 |.endif | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 |.if not(X64) | add r4, 12 |.endif zend_jit_check_exception(Dst); | jmp >9 |.code |1: } } var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_DIV: if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 1 /* may overflow */, may_throw)) { return 0; } break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, NULL, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, op1_data_range, 0, var_addr, var_def_info, var_info, may_throw)) { return 0; } break; case ZEND_CONCAT: if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, may_throw)) { return 0; } break; default: ZEND_UNREACHABLE(); } } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { binary_op_type binary_op; if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |.cold_code |7: } | SET_EX_OPLINE opline, r0 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_UNUSED) { | xor FCARG2a, FCARG2a } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } binary_op = get_binary_op(opline->extended_value); |.if X64 | LOAD_ZVAL_ADDR CARG3, op3_addr | LOAD_ADDR CARG4, binary_op |.else | sub r4, 8 | PUSH_ADDR binary_op, r0 | PUSH_ZVAL_ADDR op3_addr, r0 |.endif | EXT_CALL zend_jit_assign_dim_op_helper, r0 |.if not(X64) | add r4, 8 |.endif if (!zend_jit_check_exception(Dst)) { return 0; } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { | jmp >9 // END |.code } } |9: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline return 1; } static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) { zend_jit_addr op1_addr, op2_addr; ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); op1_addr = OP1_ADDR(); op2_addr = OP2_ADDR(); if (op1_info & MAY_BE_REF) { binary_op_type binary_op = get_binary_op(opline->extended_value); | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 | GET_Z_PTR FCARG1a, FCARG1a | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 | jnz >2 | add FCARG1a, offsetof(zend_reference, val) |.cold_code |2: | LOAD_ZVAL_ADDR FCARG2a, op2_addr |.if X64 | LOAD_ADDR CARG3, binary_op |.else | sub r4, 12 | PUSH_ADDR binary_op, r0 |.endif | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 |.if not(X64) | add r4, 12 |.endif zend_jit_check_exception(Dst); | jmp >9 |.code |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } int result; switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_DIV: result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); break; case ZEND_CONCAT: result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); break; default: ZEND_UNREACHABLE(); } |9: return result; } static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, zend_ssa_range *op1_range, zend_jit_addr op1_addr, zend_ssa_range *op2_range, zend_jit_addr op2_addr, bool *result) { zend_long op1_min; zend_long op1_max; zend_long op2_min; zend_long op2_max; if (op1_range) { op1_min = op1_range->min; op1_max = op1_range->max; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG); op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr)); } else { return 0; } if (op2_range) { op2_min = op2_range->min; op2_max = op2_range->max; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG); op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr)); } else { return 0; } switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { *result = 1; return 1; } else if (op1_max < op2_min || op1_min > op2_max) { *result = 0; return 1; } return 0; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { *result = 0; return 1; } else if (op1_max < op2_min || op1_min > op2_max) { *result = 1; return 1; } return 0; case ZEND_IS_SMALLER: if (op1_max < op2_min) { *result = 1; return 1; } else if (op1_min >= op2_max) { *result = 0; return 1; } return 0; case ZEND_IS_SMALLER_OR_EQUAL: if (op1_max <= op2_min) { *result = 1; return 1; } else if (op1_min > op2_max) { *result = 0; return 1; } return 0; default: ZEND_UNREACHABLE(); } return 0; } static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_ssa_range *op1_range, zend_jit_addr op1_addr, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { bool swap = 0; bool result; if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE) } if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { if (!result) { | jmp => target_label } } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { if (result) { | jmp => target_label } } else if (smart_branch_opcode == ZEND_JMPZNZ) { if (!result) { | jmp => target_label } else { | jmp => target_label2 } } else { ZEND_UNREACHABLE(); } } return 1; } if (skip_comparison) { if (Z_MODE(op1_addr) != IS_REG && (Z_MODE(op2_addr) == IS_REG || (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { swap = 1; } } else if (Z_MODE(op1_addr) == IS_REG) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) } else { | LONG_OP cmp, Z_REG(op1_addr), op2_addr } } else if (Z_MODE(op2_addr) == IS_REG) { if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) } else { | LONG_OP cmp, Z_REG(op2_addr), op1_addr } swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) swap = 1; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) } else { | GET_ZVAL_LVAL ZREG_R0, op1_addr if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { | test r0, r0 } else { | LONG_OP cmp, ZREG_R0, op2_addr } } if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | sete al break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | setne al break; case ZEND_IS_SMALLER: if (swap) { | setg al } else { | setl al } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | setge al } else { | setle al } break; default: ZEND_UNREACHABLE(); } | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax } if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { | jne &exit_addr } else { | jne => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | je &exit_addr } else { | je => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { | jne &exit_addr } else { | je => target_label } break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { | jle &exit_addr } else { | jle => target_label } } else { if (exit_addr) { | jge &exit_addr } else { | jge => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { | jl &exit_addr } else { | jl => target_label } } else { if (exit_addr) { | jg &exit_addr } else { | jg => target_label } } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { | je &exit_addr } else { | je => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | jne &exit_addr } else { | jne => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { | je &exit_addr } else { | jne => target_label } break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { | jg &exit_addr } else { | jg => target_label } } else { if (exit_addr) { | jl &exit_addr } else { | jl => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { | jge &exit_addr } else { | jge => target_label } } else { if (exit_addr) { | jle &exit_addr } else { | jle => target_label } } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | jne => target_label break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | je => target_label break; case ZEND_IS_SMALLER: if (swap) { | jle => target_label } else { | jge => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | jl => target_label } else { | jg => target_label } break; default: ZEND_UNREACHABLE(); } | jmp => target_label2 } else { ZEND_UNREACHABLE(); } } else { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | sete al break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | setne al break; case ZEND_IS_SMALLER: if (swap) { | setg al } else { | setl al } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | setge al } else { | setle al } break; default: ZEND_UNREACHABLE(); } | movzx eax, al | add eax, 2 | SET_ZVAL_TYPE_INFO res_addr, eax } return 1; } static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { | jne &exit_addr | jp &exit_addr } else { | jne => target_label | jp => target_label } break; case ZEND_IS_NOT_EQUAL: | jp >1 if (exit_addr) { | je &exit_addr } else { | je => target_label } |1: break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { | jne &exit_addr | jp &exit_addr } else { | jp >1 | je => target_label |1: } break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { | jbe &exit_addr } else { | jbe => target_label } } else { if (exit_addr) { | jae &exit_addr | jp &exit_addr } else { | jae => target_label | jp => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { | jb &exit_addr } else { | jb => target_label } } else { if (exit_addr) { | ja &exit_addr | jp &exit_addr } else { | ja => target_label | jp => target_label } } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | jp >1 if (exit_addr) { | je &exit_addr } else { | je => target_label } |1: break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | jne &exit_addr | jp &exit_addr } else { | jne => target_label | jp => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { | jp >1 | je &exit_addr |1: } else { | jne => target_label | jp => target_label } break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { | ja &exit_addr } else { | ja => target_label } } else { | jp >1 if (exit_addr) { | jb &exit_addr } else { | jb => target_label } |1: } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { | jae &exit_addr } else { | jae => target_label } } else { | jp >1 if (exit_addr) { | jbe &exit_addr } else { | jbe => target_label } |1: } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | jne => target_label | jp => target_label break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | jp => target_label2 | je => target_label break; case ZEND_IS_SMALLER: if (swap) { | jbe => target_label } else { | jae => target_label | jp => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | jb => target_label } else { | ja => target_label | jp => target_label } break; default: ZEND_UNREACHABLE(); } | jmp => target_label2 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jne => target_label | jp => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | jp >1 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | je => target_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE break; case ZEND_IS_SMALLER: if (swap) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jbe => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jae => target_label | jp => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jb => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | ja => target_label | jp => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | jp >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | je => target_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jne => target_label | jp => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE break; case ZEND_IS_SMALLER: if (swap) { | seta al | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax | ja => target_label } else { | jp >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jb => target_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | setae al | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax | jae => target_label } else { | jp >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jbe => target_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } break; default: ZEND_UNREACHABLE(); } } else { ZEND_UNREACHABLE(); } } else { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | jp >1 | mov eax, IS_TRUE | je >2 |1: | mov eax, IS_FALSE |2: break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | jp >1 | mov eax, IS_FALSE | je >2 |1: | mov eax, IS_TRUE |2: break; case ZEND_IS_SMALLER: if (swap) { | seta al | movzx eax, al | add eax, 2 } else { | jp >1 | mov eax, IS_TRUE | jb >2 |1: | mov eax, IS_FALSE |2: } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | setae al | movzx eax, al | add eax, 2 } else { | jp >1 | mov eax, IS_TRUE | jbe >2 |1: | mov eax, IS_FALSE |2: } break; default: ZEND_UNREACHABLE(); } | SET_ZVAL_TYPE_INFO res_addr, eax } return 1; } static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0 | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); } static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0 | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); } static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { bool swap = 0; if (Z_MODE(op1_addr) == IS_REG) { | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr } else if (Z_MODE(op2_addr) == IS_REG) { | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr swap = 1; } else { zend_reg tmp_reg = ZREG_XMM0; | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr } return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); } static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { | LONG_OP_WITH_CONST cmp, res_addr, Z_L(0) if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: | sete al break; case ZEND_IS_NOT_EQUAL: | setne al break; case ZEND_IS_SMALLER: | setl al break; case ZEND_IS_SMALLER_OR_EQUAL: | setle al break; default: ZEND_UNREACHABLE(); } | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax } if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: if (exit_addr) { | jne &exit_addr } else { | jne => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | je &exit_addr } else { | je => target_label } break; case ZEND_IS_SMALLER: if (exit_addr) { | jge &exit_addr } else { | jge => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (exit_addr) { | jg &exit_addr } else { | jg => target_label } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: if (exit_addr) { | je &exit_addr } else { | je => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | jne &exit_addr } else { | jne => target_label } break; case ZEND_IS_SMALLER: if (exit_addr) { | jl &exit_addr } else { | jl => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (exit_addr) { | jle &exit_addr } else { | jle => target_label } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: | jne => target_label break; case ZEND_IS_NOT_EQUAL: | je => target_label break; case ZEND_IS_SMALLER: | jge => target_label break; case ZEND_IS_SMALLER_OR_EQUAL: | jg => target_label break; default: ZEND_UNREACHABLE(); } | jmp => target_label2 } else { ZEND_UNREACHABLE(); } } else { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: | sete al break; case ZEND_IS_NOT_EQUAL: | setne al break; case ZEND_IS_SMALLER: | setl al break; case ZEND_IS_SMALLER_OR_EQUAL: | setle al break; default: ZEND_UNREACHABLE(); } | movzx eax, al | add eax, 2 | SET_ZVAL_TYPE_INFO res_addr, eax } return 1; } static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); bool has_slow; has_slow = (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 |.cold_code |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | jmp >6 |.code } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } } if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { |.cold_code |4: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | jmp >6 } if (!same_ops) { |5: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | jmp >6 } |.code } } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } } if (!same_ops && (op2_info & MAY_BE_LONG)) { if (op2_info & MAY_BE_DOUBLE) { |.cold_code } |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 } if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { | jmp >6 |.code } } } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } } if (!same_ops && (op1_info & MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { |.cold_code } |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 } if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { | jmp >6 |.code } } } if (has_slow || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { if (has_slow) { |.cold_code |9: } | SET_EX_OPLINE opline, r0 if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2a, op1_addr if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1 | mov FCARG1a, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval |1: } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 | mov T1, FCARG2a // save | mov FCARG1a, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 | mov FCARG2a, T1 // restore |.if X64 | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval |.else | sub r4, 12 | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 |.endif | jmp >2 |1: |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif |2: } else { |.if X64 | LOAD_ZVAL_ADDR CARG3, op2_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR op2_addr, r0 |.endif } | LOAD_ZVAL_ADDR FCARG1a, res_addr | EXT_CALL compare_function, r0 |.if not(X64) | add r4, 12 |.endif if (opline->opcode != ZEND_CASE) { | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline } | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (has_slow) { | jmp >6 |.code } } |6: return 1; } static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { uint32_t identical_label = (uint32_t)-1; uint32_t not_identical_label = (uint32_t)-1; if (smart_branch_opcode && !exit_addr) { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { if (smart_branch_opcode == ZEND_JMPZ) { not_identical_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { identical_label = target_label; } else if (smart_branch_opcode == ZEND_JMPZNZ) { not_identical_label = target_label; identical_label = target_label2; } else { ZEND_UNREACHABLE(); } } else { if (smart_branch_opcode == ZEND_JMPZ) { identical_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { not_identical_label = target_label; } else if (smart_branch_opcode == ZEND_JMPZNZ) { identical_label = target_label; not_identical_label = target_label2; } else { ZEND_UNREACHABLE(); } } } if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { return 0; } return 1; } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } return 1; } if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { op1_info |= MAY_BE_NULL; op2_info |= MAY_BE_NULL; | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 |.cold_code |1: | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, r0 | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval | jmp >1 |.code |1: | LOAD_ZVAL_ADDR FCARG2a, op2_addr | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 |.cold_code |1: | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, r0 | mov aword T1, FCARG1a // save | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | mov FCARG1a, aword T1 // restore | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | jmp >1 |.code |1: } else if (op1_info & MAY_BE_UNDEF) { op1_info |= MAY_BE_NULL; | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 |.cold_code |1: | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, r0 | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval | jmp >1 |.code |1: if (opline->op2_type != IS_CONST) { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } } else if (op2_info & MAY_BE_UNDEF) { op2_info |= MAY_BE_NULL; | LOAD_ZVAL_ADDR FCARG2a, op2_addr | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 |.cold_code |1: | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, r0 | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | jmp >1 |.code |1: if (opline->op1_type != IS_CONST) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { if (opline->op1_type != IS_CONST) { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type != IS_CONST) { if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2a, op2_addr } } if ((op1_info & op2_info & MAY_BE_ANY) == 0) { if ((opline->opcode != ZEND_CASE_STRICT && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { | SET_EX_OPLINE opline, r0 if (opline->opcode != ZEND_CASE_STRICT) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline } | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline } if (smart_branch_opcode) { if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } } else { | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) if (may_throw) { zend_jit_check_exception(Dst); } } return 1; } if (opline->op1_type & (IS_CV|IS_VAR)) { | ZVAL_DEREF FCARG1a, op1_info } if (opline->op2_type & (IS_CV|IS_VAR)) { | ZVAL_DEREF FCARG2a, op2_info } if (has_concrete_type(op1_info) && has_concrete_type(op2_info) && concrete_type(op1_info) == concrete_type(op2_info) && concrete_type(op1_info) <= IS_TRUE) { if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jmp &exit_addr } } else if (identical_label != (uint32_t)-1) { | jmp =>identical_label } } else { | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) } } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jmp &exit_addr } } else if (identical_label != (uint32_t)-1) { | jmp =>identical_label } } else { | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) } } else { if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } } else { | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) } } } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { zval *val = Z_ZV(op1_addr); | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) if (smart_branch_opcode) { if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { | jne >8 | SET_EX_OPLINE opline, r0 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { | jmp &exit_addr } else if (identical_label != (uint32_t)-1) { | jmp =>identical_label } else { | jmp >9 } |8: } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { | je &exit_addr } else if (identical_label != (uint32_t)-1) { | je =>identical_label } else { | je >9 } } else { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { | sete al } else { | setne al } | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax } if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | SET_EX_OPLINE opline, r0 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp &exit_addr } } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { zval *val = Z_ZV(op2_addr); | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) if (smart_branch_opcode) { if (opline->opcode != ZEND_CASE_STRICT && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { | jne >8 | SET_EX_OPLINE opline, r0 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { | jmp &exit_addr } else if (identical_label != (uint32_t)-1) { | jmp =>identical_label } else { | jmp >9 } |8: } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { | je &exit_addr } else if (identical_label != (uint32_t)-1) { | je =>identical_label } else { | je >9 } } else { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { | sete al } else { | setne al } | movzx eax, al | lea eax, [eax + 2] | SET_ZVAL_TYPE_INFO res_addr, eax } if (opline->opcode != ZEND_CASE_STRICT && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | SET_EX_OPLINE opline, r0 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } } if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | jmp =>not_identical_label } } } else { if (opline->op1_type == IS_CONST) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_CONST) { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } | EXT_CALL zend_is_identical, r0 if ((opline->opcode != ZEND_CASE_STRICT && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { | mov aword T1, r0 // save | SET_EX_OPLINE opline, r0 if (opline->opcode != ZEND_CASE_STRICT) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline } | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | mov r0, aword T1 // restore } if (smart_branch_opcode) { | test al, al if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jnz &exit_addr } else { | jz &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | jz =>not_identical_label if (identical_label != (uint32_t)-1) { | jmp =>identical_label } } else if (identical_label != (uint32_t)-1) { | jnz =>identical_label } } else { | movzx eax, al if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { | lea eax, [eax + 2] } else { | neg eax | lea eax, [eax + 3] } | SET_ZVAL_TYPE_INFO res_addr, eax } } |9: if (may_throw) { zend_jit_check_exception(Dst); } return 1; } static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) { uint32_t true_label = -1; uint32_t false_label = -1; bool set_bool = 0; bool set_bool_not = 0; bool set_delayed = 0; bool jmp_done = 0; if (branch_opcode == ZEND_BOOL) { set_bool = 1; } else if (branch_opcode == ZEND_BOOL_NOT) { set_bool = 1; set_bool_not = 1; } else if (branch_opcode == ZEND_JMPZ) { false_label = target_label; } else if (branch_opcode == ZEND_JMPNZ) { true_label = target_label; } else if (branch_opcode == ZEND_JMPZNZ) { true_label = target_label2; false_label = target_label; } else if (branch_opcode == ZEND_JMPZ_EX) { set_bool = 1; false_label = target_label; } else if (branch_opcode == ZEND_JMPNZ_EX) { set_bool = 1; true_label = target_label; } else { ZEND_UNREACHABLE(); } if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { if (zend_is_true(Z_ZV(op1_addr))) { /* Always TRUE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } } if (true_label != (uint32_t)-1) { | jmp =>true_label; } } else { /* Always FALSE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } } if (false_label != (uint32_t)-1) { | jmp =>false_label; } } return 1; } if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { /* Always TRUE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } } if (true_label != (uint32_t)-1) { | jmp =>true_label; } } else { if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { /* Always FALSE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } } } else { | CMP_ZVAL_TYPE op1_addr, IS_TRUE if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { if ((op1_info & MAY_BE_LONG) && !(op1_info & MAY_BE_UNDEF) && !set_bool) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ) { | jl >9 } else { | jl &exit_addr } } else if (false_label != (uint32_t)-1) { | jl =>false_label } else { | jl >9 } jmp_done = 1; } else { | jg >2 } } if (!(op1_info & MAY_BE_TRUE)) { /* It's FALSE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } } } else { if (exit_addr) { if (set_bool) { | jne >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | jmp &exit_addr } else { | jmp >9 } |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { | jne &exit_addr } } } else { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | je &exit_addr } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { | jne &exit_addr } else { | je >9 } } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (set_bool) { | jne >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE if (true_label != (uint32_t)-1) { | jmp =>true_label } else { | jmp >9 } |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } else { if (true_label != (uint32_t)-1) { | je =>true_label } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { | jne =>false_label jmp_done = 1; } else { | je >9 } } } else if (set_bool) { | sete al | movzx eax, al if (set_bool_not) { | neg eax | add eax, 3 } else { | add eax, 2 } if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { set_delayed = 1; } else { | SET_ZVAL_TYPE_INFO res_addr, eax } } } } /* It's FALSE, but may be UNDEF */ if (op1_info & MAY_BE_UNDEF) { if (op1_info & MAY_BE_ANY) { if (set_delayed) { | CMP_ZVAL_TYPE op1_addr, IS_UNDEF | SET_ZVAL_TYPE_INFO res_addr, eax | jz >1 } else { | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 } |.cold_code |1: } | mov FCARG1d, opline->op1.var | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_undefined_op_helper, r0 if (may_throw) { if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } if (exit_addr) { if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { | jmp &exit_addr } } else if (false_label != (uint32_t)-1) { | jmp =>false_label } if (op1_info & MAY_BE_ANY) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | jmp >9 } } else if (false_label == (uint32_t)-1) { | jmp >9 } |.code } } if (!jmp_done) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { if (op1_info & MAY_BE_LONG) { | jmp >9 } } else if (op1_info & MAY_BE_LONG) { | jmp &exit_addr } } else if (false_label != (uint32_t)-1) { | jmp =>false_label } else if (op1_info & MAY_BE_LONG) { | jmp >9 } } } } if (op1_info & MAY_BE_LONG) { |2: if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 } if (Z_MODE(op1_addr) == IS_REG) { | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) } else { | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) } if (set_bool) { | setne al | movzx eax, al if (set_bool_not) { | neg eax | add eax, 3 } else { | lea eax, [eax + 2] } | SET_ZVAL_TYPE_INFO res_addr, eax } if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | jne &exit_addr } else { | je &exit_addr } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (true_label != (uint32_t)-1) { | jne =>true_label if (false_label != (uint32_t)-1) { | jmp =>false_label } } else { | je =>false_label } } } if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { if (CAN_USE_AVX()) { | vxorps xmm0, xmm0, xmm0 } else { | xorps xmm0, xmm0 } | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr if (set_bool) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | jp >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jne &exit_addr |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jp &exit_addr | je &exit_addr | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } } else if (false_label != (uint32_t)-1) { // JMPZ_EX | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jp >1 | je => false_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE |1: } else if (true_label != (uint32_t)-1) { // JMPNZ_EX | jp >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jne => true_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } else if (set_bool_not) { // BOOL_NOT | jp >1 | mov eax, IS_TRUE | je >2 |1: | mov eax, IS_FALSE |2: | SET_ZVAL_TYPE_INFO res_addr, eax } else { // BOOL | jp >1 | mov eax, IS_TRUE | jne >2 |1: | mov eax, IS_FALSE |2: | SET_ZVAL_TYPE_INFO res_addr, eax } } else { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | jp >1 | jne &exit_addr |1: } else { | jp &exit_addr | je &exit_addr } } else { ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); if (false_label != (uint32_t)-1) { | jp =>false_label } else { | jp >1 } if (true_label != (uint32_t)-1) { | jne =>true_label if (false_label != (uint32_t)-1) { | jmp =>false_label } } else { | je =>false_label } |1: } } } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.cold_code |2: } if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | SET_EX_OPLINE opline, r0 | EXT_CALL zend_is_true, r0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 } | GET_ZVAL_PTR FCARG1a, op1_addr | GC_DELREF FCARG1a | jnz >3 | mov aword T1, r0 // save | ZVAL_DTOR_FUNC op1_info, opline | mov r0, aword T1 // restore |3: } if (may_throw) { | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 | jne ->exception_handler_undef } if (set_bool) { if (set_bool_not) { | neg eax | add eax, 3 } else { | add eax, 2 } | SET_ZVAL_TYPE_INFO res_addr, eax if (exit_addr) { | CMP_ZVAL_TYPE res_addr, IS_FALSE if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | jne &exit_addr } else { | je &exit_addr } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { | CMP_ZVAL_TYPE res_addr, IS_FALSE if (true_label != (uint32_t)-1) { | jne =>true_label if (false_label != (uint32_t)-1) { | jmp =>false_label } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | jmp >9 } } else { | je =>false_label } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | jmp >9 |.code } } else { | test r0, r0 if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | jne &exit_addr if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | jmp >9 } } else { | je &exit_addr if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | jmp >9 } } } else if (true_label != (uint32_t)-1) { | jne =>true_label if (false_label != (uint32_t)-1) { | jmp =>false_label } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | jmp >9 } } else { | je =>false_label if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | jmp >9 } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.code } } } |9: return 1; } static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) { if (op1_addr != op1_def_addr) { if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { return 0; } if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { op1_addr = op1_def_addr; } } if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) { return 0; } if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } return 1; } static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) { ZEND_ASSERT(opline->op1_type == IS_CV); if (op2_addr != op2_def_addr) { if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { return 0; } if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { op2_addr = op2_def_addr; } } if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op1_use_addr) == IS_REG && !Z_LOAD(op1_use_addr) && !Z_STORE(op1_use_addr)) { /* Force type update */ op1_info |= MAY_BE_UNDEF; } if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, may_throw)) { return 0; } if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { return 0; } if (opline->result_type != IS_UNUSED) { if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } } return 1; } /* copy of hidden zend_closure */ typedef struct _zend_closure { zend_object std; zend_function func; zval this_ptr; zend_class_entry *called_scope; zif_handler orig_internal_handler; } zend_closure; static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | // Check Stack Overflow | MEM_OP2_2_ZTS mov, r1, aword, executor_globals, vm_stack_end, r0 | MEM_OP2_2_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0 | cmp r1, used_stack | jb &exit_addr return 1; } static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool use_this, bool stack_check) { uint32_t used_stack; if (func) { used_stack = zend_vm_calc_used_stack(opline->extended_value, func); } else { used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); | // if (EXPECTED(ZEND_USER_CODE(func->type))) { if (!is_closure) { | test byte [r0 + offsetof(zend_function, type)], 1 | mov FCARG1a, used_stack | jnz >1 } else { | mov FCARG1a, used_stack } | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); | mov edx, opline->extended_value if (!is_closure) { | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] } else { | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)] | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)] } | shl edx, 5 |.if X64 | movsxd r2, edx |.endif | sub FCARG1a, r2 |1: } zend_jit_start_reuse_ip(); | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX if (stack_check) { | // Check Stack Overflow | MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2 | sub r2, RX if (func) { | cmp r2, used_stack } else { | cmp r2, FCARG1a } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | jb &exit_addr } else { | jb >1 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); |.cold_code |1: if (func) { | mov FCARG1d, used_stack } #ifdef _WIN32 if (0) { #else if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { #endif | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_int_extend_stack_helper, r0 } else { if (!is_closure) { | mov FCARG2a, r0 } else { | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] } | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_extend_stack_helper, r0 } | mov RX, r0 | jmp >1 |.code } } if (func) { | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 } else { | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 } | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { | // ZEND_SET_CALL_INFO(call, 0, call_info); | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) } #ifdef _WIN32 if (0) { #else if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { #endif | // call->func = func; |1: | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 } else { if (!is_closure) { | // call->func = func; if (func && op_array == &func->op_array && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 } else { | mov aword EX:RX->func, r0 } } else { | // call->func = &closure->func; | lea r1, aword [r0 + offsetof(zend_closure, func)] | mov aword EX:RX->func, r1 } |1: } if (opline->opcode == ZEND_INIT_METHOD_CALL) { | // Z_PTR(call->This) = obj; | mov r1, aword T1 | mov aword EX:RX->This.value.ptr, r1 if (opline->op1_type == IS_UNUSED || use_this) { | // call->call_info |= ZEND_CALL_HAS_THIS; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS } else { | or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS } } else { if (opline->op1_type == IS_CV) { | // GC_ADDREF(obj); | add dword [r1], 1 } | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) } else { | or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) } } } else if (!is_closure) { | // Z_CE(call->This) = called_scope; | mov aword EX:RX->This.value.ptr, 0 } else { if (opline->op2_type == IS_CV) { | // GC_ADDREF(closure); | add dword [r0], 1 } | // object_or_called_scope = closure->called_scope; | mov r1, aword [r0 + offsetof(zend_closure, called_scope)] | mov aword EX:RX->This.value.ptr, r1 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)] | and edx, ZEND_ACC_FAKE_CLOSURE | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF | jz >1 | // call_info |= ZEND_CALL_HAS_THIS; | or edx, ZEND_CALL_HAS_THIS | // object_or_called_scope = Z_OBJ(closure->this_ptr); | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)] |1: | // ZEND_SET_CALL_INFO(call, 0, call_info); | or dword EX:RX->This.u1.type_info, edx | // Z_PTR(call->This) = object_or_called_scope; | mov aword EX:RX->This.value.ptr, r1 | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0 | jnz >1 | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)] | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 |1: } | // ZEND_CALL_NUM_ARGS(call) = num_args; | mov dword EX:RX->This.u2.num_args, opline->extended_value return 1; } static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, zend_jit_trace_rec *trace) { int skip; if (trace) { zend_jit_trace_rec *p = trace; ssa_op++; while (1) { if (p->op == ZEND_JIT_TRACE_VM) { switch (p->opline->opcode) { case ZEND_SEND_ARRAY: case ZEND_SEND_USER: case ZEND_SEND_UNPACK: case ZEND_INIT_FCALL: case ZEND_INIT_METHOD_CALL: case ZEND_INIT_STATIC_METHOD_CALL: case ZEND_INIT_FCALL_BY_NAME: case ZEND_INIT_NS_FCALL_BY_NAME: case ZEND_INIT_DYNAMIC_CALL: case ZEND_NEW: case ZEND_INIT_USER_CALL: case ZEND_FAST_CALL: case ZEND_JMP: case ZEND_JMPZNZ: case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: case ZEND_FE_RESET_R: case ZEND_FE_RESET_RW: case ZEND_JMP_SET: case ZEND_COALESCE: case ZEND_JMP_NULL: case ZEND_ASSERT_CHECK: case ZEND_CATCH: case ZEND_DECLARE_ANON_CLASS: case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: return 1; case ZEND_DO_ICALL: case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: case ZEND_DO_FCALL: return 0; case ZEND_SEND_VAL: case ZEND_SEND_VAR: case ZEND_SEND_VAL_EX: case ZEND_SEND_VAR_EX: case ZEND_SEND_FUNC_ARG: case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: /* skip */ break; default: if (zend_may_throw(opline, ssa_op, op_array, ssa)) { return 1; } } ssa_op += zend_jit_trace_op_len(opline); } else if (p->op == ZEND_JIT_TRACE_ENTER || p->op == ZEND_JIT_TRACE_BACK || p->op == ZEND_JIT_TRACE_END) { return 1; } p++; } } if (!call_info) { const zend_op *end = op_array->opcodes + op_array->last; opline++; ssa_op++; skip = 1; while (opline != end) { if (!skip) { if (zend_may_throw(opline, ssa_op, op_array, ssa)) { return 1; } } switch (opline->opcode) { case ZEND_SEND_VAL: case ZEND_SEND_VAR: case ZEND_SEND_VAL_EX: case ZEND_SEND_VAR_EX: case ZEND_SEND_FUNC_ARG: case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: skip = 0; break; case ZEND_SEND_ARRAY: case ZEND_SEND_USER: case ZEND_SEND_UNPACK: case ZEND_INIT_FCALL: case ZEND_INIT_METHOD_CALL: case ZEND_INIT_STATIC_METHOD_CALL: case ZEND_INIT_FCALL_BY_NAME: case ZEND_INIT_NS_FCALL_BY_NAME: case ZEND_INIT_DYNAMIC_CALL: case ZEND_NEW: case ZEND_INIT_USER_CALL: case ZEND_FAST_CALL: case ZEND_JMP: case ZEND_JMPZNZ: case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: case ZEND_FE_RESET_R: case ZEND_FE_RESET_RW: case ZEND_JMP_SET: case ZEND_COALESCE: case ZEND_JMP_NULL: case ZEND_ASSERT_CHECK: case ZEND_CATCH: case ZEND_DECLARE_ANON_CLASS: case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: return 1; case ZEND_DO_ICALL: case ZEND_DO_UCALL: case ZEND_DO_FCALL_BY_NAME: case ZEND_DO_FCALL: end = opline; if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { /* INIT_FCALL and DO_FCALL in different BasicBlocks */ return 1; } return 0; } opline++; ssa_op++; } return 1; } else { const zend_op *end = call_info->caller_call_opline; if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { /* INIT_FCALL and DO_FCALL in different BasicBlocks */ return 1; } opline++; ssa_op++; skip = 1; while (opline != end) { if (skip) { switch (opline->opcode) { case ZEND_SEND_VAL: case ZEND_SEND_VAR: case ZEND_SEND_VAL_EX: case ZEND_SEND_VAR_EX: case ZEND_SEND_FUNC_ARG: case ZEND_SEND_REF: case ZEND_SEND_VAR_NO_REF: case ZEND_SEND_VAR_NO_REF_EX: skip = 0; break; case ZEND_SEND_ARRAY: case ZEND_SEND_USER: case ZEND_SEND_UNPACK: return 1; } } else { if (zend_may_throw(opline, ssa_op, op_array, ssa)) { return 1; } } opline++; ssa_op++; } return 0; } } static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) { int32_t exit_point; const void *exit_addr; if (func->type == ZEND_INTERNAL_FUNCTION) { #ifdef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? return 0; #endif } else if (func->type == ZEND_USER_FUNCTION) { if (!zend_accel_in_shm(func->op_array.opcodes)) { /* op_array and op_array->opcodes are not persistent. We can't link. */ return 0; } } else { ZEND_UNREACHABLE(); return 0; } exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | // call = EX(call); | mov r1, EX->call while (level > 0) { | mov r1, EX:r1->prev_execute_data level--; } if (func->type == ZEND_USER_FUNCTION && (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || (func->common.fn_flags & ZEND_ACC_CLOSURE) || !func->common.function_name)) { const zend_op *opcodes = func->op_array.opcodes; | mov r1, aword EX:r1->func | .if X64 || if (!IS_SIGNED_32BIT(opcodes)) { | mov64 r2, ((ptrdiff_t)opcodes) | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2 || } else { | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes || } | .else | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes | .endif | jne &exit_addr } else { | .if X64 || if (!IS_SIGNED_32BIT(func)) { | mov64 r2, ((ptrdiff_t)func) | cmp aword EX:r1->func, r2 || } else { | cmp aword EX:r1->func, func || } | .else | cmp aword EX:r1->func, func | .endif | jne &exit_addr } return 1; } static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, bool stack_check) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } } if (info) { call_info = info->callee_info; while (call_info && call_info->caller_init_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } } if (!func && trace && trace->op == ZEND_JIT_TRACE_INIT_CALL) { #ifdef _WIN32 /* ASLR */ if (trace->func->type != ZEND_INTERNAL_FUNCTION) { func = (zend_function*)trace->func; } #else func = (zend_function*)trace->func; #endif } #ifdef _WIN32 if (0) { #else if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { #endif /* load constant address later */ } else if (func && op_array == &func->op_array) { /* recursive call */ if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) || (sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) { | mov r0, EX->func } } else { | // if (CACHED_PTR(opline->result.num)) | mov r0, EX->run_time_cache | mov r0, aword [r0 + opline->result.num] | test r0, r0 | jz >1 |.cold_code |1: if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { | LOAD_ADDR FCARG1a, func | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 | mov r1, EX->run_time_cache | mov aword [r1 + opline->result.num], r0 | jmp >3 } else { zval *zv = RT_CONSTANT(opline, opline->op2); if (opline->opcode == ZEND_INIT_FCALL) { | LOAD_ADDR FCARG1a, Z_STR_P(zv); | EXT_CALL zend_jit_find_func_helper, r0 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1); | EXT_CALL zend_jit_find_func_helper, r0 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { | LOAD_ADDR FCARG1a, zv; | EXT_CALL zend_jit_find_ns_func_helper, r0 } else { ZEND_UNREACHABLE(); } | // CACHE_PTR(opline->result.num, fbc); | mov r1, EX->run_time_cache | mov aword [r1 + opline->result.num], r0 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } if (!func || opline->opcode == ZEND_INIT_FCALL) { | test r0, r0 | jnz >3 } else if (func->type == ZEND_USER_FUNCTION && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { const zend_op *opcodes = func->op_array.opcodes; | .if X64 || if (!IS_SIGNED_32BIT(opcodes)) { | mov64 r1, ((ptrdiff_t)opcodes) | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 || } else { | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes || } | .else | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes | .endif | jz >3 } else { | .if X64 || if (!IS_SIGNED_32BIT(func)) { | mov64 r1, ((ptrdiff_t)func) | cmp r0, r1 || } else { | cmp r0, func || } | .else | cmp r0, func | .endif | jz >3 } | jmp &exit_addr } else { | test r0, r0 | jnz >3 | // SAVE_OPLINE(); | SET_EX_OPLINE opline, r0 | jmp ->undefined_function } } |.code |3: } if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, stack_check)) { return 0; } if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, trace)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } } else { delayed_call_chain = 1; delayed_call_level = call_level; } return 1; } static int zend_jit_init_method_call(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, uint32_t op1_info, zend_jit_addr op1_addr, zend_class_entry *ce, bool ce_is_instanceof, bool use_this, zend_class_entry *trace_ce, zend_jit_trace_rec *trace, bool stack_check, bool polymorphic_side_trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; zval *function_name; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); function_name = RT_CONSTANT(opline, opline->op2); if (info) { call_info = info->callee_info; while (call_info && call_info->caller_init_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } } if (polymorphic_side_trace) { /* function is passed in r0 from parent_trace */ } else { if (opline->op1_type == IS_UNUSED || use_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); | GET_ZVAL_PTR FCARG1a, this_addr } else { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | ZVAL_DEREF FCARG1a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } else { /* Hack: Convert reference to regular value to simplify JIT code */ ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 | LOAD_ZVAL_ADDR FCARG1a, op1_addr | EXT_CALL zend_jit_unref_helper, r0 |1: } } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 |.cold_code |1: if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | SET_EX_OPLINE opline, r0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { | EXT_CALL zend_jit_invalid_method_call_tmp, r0 } else { | EXT_CALL zend_jit_invalid_method_call, r0 } | jmp ->exception_handler |.code } } | GET_ZVAL_PTR FCARG1a, op1_addr } if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } } | mov aword T1, FCARG1a // save if (func) { | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); | mov r0, EX->run_time_cache | mov r0, aword [r0 + opline->result.num + sizeof(void*)] | test r0, r0 | jz >1 } else { | // if (CACHED_PTR(opline->result.num) == obj->ce)) { | mov r0, EX->run_time_cache | mov r2, aword [r0 + opline->result.num] | cmp r2, [FCARG1a + offsetof(zend_object, ce)] | jnz >1 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); | mov r0, aword [r0 + opline->result.num + sizeof(void*)] } |.cold_code |1: | LOAD_ADDR FCARG2a, function_name |.if X64 | lea CARG3, aword T1 |.else | lea r0, aword T1 | sub r4, 12 | push r0 |.endif | SET_EX_OPLINE opline, r0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { | EXT_CALL zend_jit_find_method_tmp_helper, r0 } else { | EXT_CALL zend_jit_find_method_helper, r0 } |.if not(X64) | add r4, 12 |.endif | test r0, r0 | jnz >2 | jmp ->exception_handler |.code |2: } if (!func && trace && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func #ifdef _WIN32 && trace->func->type != ZEND_INTERNAL_FUNCTION #endif ) { int32_t exit_point; const void *exit_addr; exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_METHOD_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } func = (zend_function*)trace->func; if (func->type == ZEND_USER_FUNCTION && (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || (func->common.fn_flags & ZEND_ACC_CLOSURE) || !func->common.function_name)) { const zend_op *opcodes = func->op_array.opcodes; | .if X64 || if (!IS_SIGNED_32BIT(opcodes)) { | mov64 r1, ((ptrdiff_t)opcodes) | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 || } else { | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes || } | .else | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes | .endif | jne &exit_addr } else { | .if X64 || if (!IS_SIGNED_32BIT(func)) { | mov64 r1, ((ptrdiff_t)func) | cmp r0, r1 || } else { | cmp r0, func || } | .else | cmp r0, func | .endif | jne &exit_addr } } if (!func) { | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC | jnz >1 |.cold_code |1: } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { | mov FCARG1a, aword T1 // restore | mov FCARG2a, r0 |.if X64 | mov CARG3d, opline->extended_value |.else | sub r4, 12 | push opline->extended_value |.endif if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0 } else { | EXT_CALL zend_jit_push_static_metod_call_frame, r0 } |.if not(X64) | add r4, 12 |.endif if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !use_this)) { | test r0, r0 | jz ->exception_handler } | mov RX, r0 } if (!func) { | jmp >9 |.code } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, use_this, stack_check)) { return 0; } } if (!func) { |9: } zend_jit_start_reuse_ip(); if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, trace)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } } else { delayed_call_chain = 1; delayed_call_level = call_level; } return 1; } static int zend_jit_init_closure_call(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, bool stack_check) { zend_function *func = NULL; zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); | GET_ZVAL_PTR r0, op2_addr if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } |.if X64 || if (!IS_SIGNED_32BIT(zend_ce_closure)) { | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure) | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a || } else { | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure || } |.else | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure |.endif | jne &exit_addr if (ssa->var_info && ssa_op->op2_use >= 0) { ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; ssa->var_info[ssa_op->op2_use].is_instanceof = 0; } } if (trace && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func && trace->func->type == ZEND_USER_FUNCTION) { const zend_op *opcodes; int32_t exit_point; const void *exit_addr; func = (zend_function*)trace->func; opcodes = func->op_array.opcodes; exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | .if X64 || if (!IS_SIGNED_32BIT(opcodes)) { | mov64 FCARG1a, ((ptrdiff_t)opcodes) | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a || } else { | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes || } | .else | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes | .endif | jne &exit_addr } if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } } if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, stack_check)) { return 0; } if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, trace)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } } else { delayed_call_chain = 1; delayed_call_level = call_level; } if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { if (!zend_jit_set_valid_ip(Dst, opline + 1)) { return 0; } } return 1; } static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) { uint32_t num_args = 0; zend_function *func = call_info->callee_func; /* It's okay to handle prototypes here, because they can only increase the accepted arguments. * Anything legal for the parent method is also legal for the parent method. */ while (num_args < call_info->num_args) { zend_arg_info *arg_info = func->op_array.arg_info + num_args; if (ZEND_TYPE_IS_SET(arg_info->type)) { if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) { zend_op *opline = call_info->arg_info[num_args].opline; zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type); if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) { break; } } else { break; } } num_args++; } return num_args; } static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; const zend_function *func = NULL; uint32_t i; zend_jit_addr res_addr; uint32_t call_num_args = 0; bool unknown_num_args = 0; const void *exit_addr = NULL; const zend_op *prev_opline; if (RETURN_VALUE_USED(opline)) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } else { /* CPU stack allocated temporary zval */ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET); } prev_opline = opline - 1; while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { prev_opline--; } if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { unknown_num_args = 1; } if (info) { call_info = info->callee_info; while (call_info && call_info->caller_call_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } } if (!func) { /* resolve function at run time */ } else if (func->type == ZEND_USER_FUNCTION) { ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); call_num_args = call_info->num_args; } else if (func->type == ZEND_INTERNAL_FUNCTION) { ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); call_num_args = call_info->num_args; } else { ZEND_UNREACHABLE(); } if (trace && !func) { if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); #ifndef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? func = trace->func; if (JIT_G(current_frame) && JIT_G(current_frame)->call && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); } else { unknown_num_args = 1; } #endif } else if (trace->op == ZEND_JIT_TRACE_ENTER) { ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); if (zend_accel_in_shm(trace->func->op_array.opcodes)) { func = trace->func; if (JIT_G(current_frame) && JIT_G(current_frame)->call && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); } else { unknown_num_args = 1; } } } } bool may_have_extra_named_params = opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); if (!reuse_ip) { zend_jit_start_reuse_ip(); | // call = EX(call); | mov RX, EX->call } zend_jit_stop_reuse_ip(); | // fbc = call->func; | // mov r2, EX:RX->func ??? | // SAVE_OPLINE(); | SET_EX_OPLINE opline, r0 if (opline->opcode == ZEND_DO_FCALL) { if (!func) { if (trace) { uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | mov r0, EX:RX->func | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED | jnz &exit_addr } } } if (!delayed_call_chain) { if (call_level == 1) { | mov aword EX->call, 0 } else { | //EX(call) = call->prev_execute_data; | mov r0, EX:RX->prev_execute_data | mov EX->call, r0 } } delayed_call_chain = 0; | //call->prev_execute_data = execute_data; | mov EX:RX->prev_execute_data, EX if (!func) { | mov r0, EX:RX->func } if (opline->opcode == ZEND_DO_FCALL) { if (!func) { if (!trace) { | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED | jnz >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG1a, RX } | EXT_CALL zend_jit_deprecated_helper, r0 | test al, al | mov r0, EX:RX->func // reload | jne >1 | jmp ->exception_handler |.code |1: } } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { if (!GCC_GLOBAL_REGS) { | mov FCARG1a, RX } | EXT_CALL zend_jit_deprecated_helper, r0 | test al, al | je ->exception_handler } } if (!func && opline->opcode != ZEND_DO_UCALL && opline->opcode != ZEND_DO_ICALL) { | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION | jne >8 } if ((!func || func->type == ZEND_USER_FUNCTION) && opline->opcode != ZEND_DO_ICALL) { | // EX(call) = NULL; | mov aword EX:RX->call, 0 if (RETURN_VALUE_USED(opline)) { | // EX(return_value) = EX_VAR(opline->result.var); | LOAD_ZVAL_ADDR r2, res_addr | mov aword EX:RX->return_value, r2 } else { | // EX(return_value) = 0; | mov aword EX:RX->return_value, 0 } //EX_LOAD_RUN_TIME_CACHE(op_array); if (!func || func->op_array.cache_size) { if (func && op_array == &func->op_array) { /* recursive call */ if (trace || func->op_array.cache_size > sizeof(void*)) { | mov r2, EX->run_time_cache | mov EX:RX->run_time_cache, r2 } } else { if (func) { | mov r0, EX:RX->func } | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] #if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR | mov r2, aword [r2] #elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 } else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) { | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 } else { /* the called op_array may be not persisted yet */ | test r2, 1 | jz >1 | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 |1: } | mov r2, aword [r2] } else { | test r2, 1 | jz >1 | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 |1: | mov r2, aword [r2] } #else # error "Unknown ZEND_MAP_PTR_KIND" #endif | mov EX:RX->run_time_cache, r2 } } | // EG(current_execute_data) = execute_data; | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 | mov FP, RX | // opline = op_array->opcodes; if (func && !unknown_num_args) { for (i = call_num_args; i < func->op_array.last_var; i++) { uint32_t n = EX_NUM_TO_VAR(i); | SET_Z_TYPE_INFO RX + n, IS_UNDEF } if (call_num_args <= func->op_array.num_args) { if (!trace || (trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { uint32_t num_args; if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { if (trace) { num_args = 0; } else if (call_info) { num_args = skip_valid_arguments(op_array, ssa, call_info); } else { num_args = call_num_args; } } else { num_args = call_num_args; } if (zend_accel_in_shm(func->op_array.opcodes)) { | LOAD_IP_ADDR (func->op_array.opcodes + num_args) } else { | mov r0, EX->func if (GCC_GLOBAL_REGS) { | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] if (num_args) { | add IP, (num_args * sizeof(zend_op)) } } else { | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] if (num_args) { | add FCARG1a, (num_args * sizeof(zend_op)) } | mov aword EX->opline, FCARG1a } } if (!trace && op_array == &func->op_array) { /* recursive call */ if (ZEND_OBSERVER_ENABLED) { | SAVE_IP | mov FCARG1a, FP | EXT_CALL zend_observer_fcall_begin, r0 } #ifdef CONTEXT_THREADED_JIT | call >1 |.cold_code |1: | pop r0 | jmp =>num_args |.code #else | jmp =>num_args #endif return 1; } } } else { if (!trace || (trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { if (func && zend_accel_in_shm(func->op_array.opcodes)) { | LOAD_IP_ADDR (func->op_array.opcodes) } else if (GCC_GLOBAL_REGS) { | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] } else { | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] | mov aword EX->opline, FCARG1a } } if (!GCC_GLOBAL_REGS) { | mov FCARG1a, FP } | EXT_CALL zend_jit_copy_extra_args_helper, r0 } } else { | // opline = op_array->opcodes if (func && zend_accel_in_shm(func->op_array.opcodes)) { | LOAD_IP_ADDR (func->op_array.opcodes) } else if (GCC_GLOBAL_REGS) { | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] } else { | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] | mov aword EX->opline, FCARG1a } if (func) { | // num_args = EX_NUM_ARGS(); | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) | cmp ecx, (func->op_array.num_args) } else { | // first_extra_arg = op_array->num_args; | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] | // num_args = EX_NUM_ARGS(); | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) | cmp ecx, edx } | jg >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG1a, FP } | EXT_CALL zend_jit_copy_extra_args_helper, r0 if (!func) { | mov r0, EX->func // reload } | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload | jmp >1 |.code if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { if (!func) { | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS | jnz >1 } | // opline += num_args; |.if X64 || ZEND_ASSERT(sizeof(zend_op) == 32); | mov edx, ecx | shl r2, 5 |.else | imul r2, ecx, sizeof(zend_op) |.endif | ADD_IP r2 } |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { if (func) { | mov edx, (func->op_array.last_var) } else { | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] } | sub edx, ecx | jle >3 //??? | // zval *var = EX_VAR_NUM(num_args); // |.if X64 // | movsxd r1, ecx // |.endif | shl r1, 4 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] |2: | SET_Z_TYPE_INFO r1, IS_UNDEF | sub edx, 1 | lea r1, [r1 + 16] | jne <2 |3: } if (ZEND_OBSERVER_ENABLED) { | SAVE_IP | mov FCARG1a, FP | EXT_CALL zend_observer_fcall_begin, r0 } if (trace) { if (!func && (opline->opcode != ZEND_DO_UCALL)) { | jmp >9 } } else { #ifdef CONTEXT_THREADED_JIT | call ->context_threaded_call if (!func && (opline->opcode != ZEND_DO_UCALL)) { | jmp >9 } | call ->context_threaded_call if (!func) { | jmp >9 } #else if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | JMP_IP } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment | JMP_IP } else { | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 1 // ZEND_VM_ENTER | ret } } #endif } if ((!func || func->type == ZEND_INTERNAL_FUNCTION) && (opline->opcode != ZEND_DO_UCALL)) { if (!func && (opline->opcode != ZEND_DO_ICALL)) { |8: } if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { if (trace) { uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED | jnz &exit_addr } else { | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED | jnz >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG1a, RX } | EXT_CALL zend_jit_deprecated_helper, r0 | test al, al | mov r0, EX:RX->func // reload | jne >1 | jmp ->exception_handler |.code |1: } } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { if (!GCC_GLOBAL_REGS) { | mov FCARG1a, RX } | EXT_CALL zend_jit_deprecated_helper, r0 | test al, al | je ->exception_handler | mov r0, EX:RX->func // reload } } | // ZVAL_NULL(EX_VAR(opline->result.var)); | LOAD_ZVAL_ADDR FCARG2a, res_addr | SET_Z_TYPE_INFO FCARG2a, IS_NULL | // EG(current_execute_data) = execute_data; | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 zend_jit_reset_last_valid_opline(); | // fbc->internal_function.handler(call, ret); | mov FCARG1a, RX if (func) { | EXT_CALL func->internal_function.handler, r0 } else { | call aword [r0 + offsetof(zend_internal_function, handler)] } | // EG(current_execute_data) = execute_data; | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 | // zend_vm_stack_free_args(call); if (func && !unknown_num_args) { for (i = 0; i < call_num_args; i++ ) { uint32_t offset = EX_NUM_TO_VAR(i); | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline } } else { | mov FCARG1a, RX | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 } if (may_have_extra_named_params) { | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) | jnz >1 |.cold_code |1: | mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)] | EXT_CALL zend_free_extra_named_params, r0 | jmp >2 |.code |2: } |8: if (opline->opcode == ZEND_DO_FCALL) { // TODO: optimize ??? | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16) | jnz >1 |.cold_code |1: | GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This) | // OBJ_RELEASE(object); | OBJ_RELEASE ZREG_FCARG1a, >2 | jmp >2 |.code |2: } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !JIT_G(current_frame)->call || !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { | // zend_vm_stack_free_call_frame(call); | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) | jnz >1 |.cold_code |1: | mov FCARG1a, RX | EXT_CALL zend_jit_free_call_frame, r0 | jmp >1 |.code } | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0 |1: if (!RETURN_VALUE_USED(opline)) { zend_class_entry *ce; bool ce_is_instanceof; uint32_t func_info = call_info ? zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); /* If an exception is thrown, the return_value may stay at the * original value of null. */ func_info |= MAY_BE_NULL; if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline } } | // if (UNEXPECTED(EG(exception) != NULL)) { | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->icall_throw_handler // TODO: Can we avoid checking for interrupts after each call ??? if (trace && last_valid_opline != opline) { int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } else { exit_addr = NULL; } if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { return 0; } if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { | LOAD_IP_ADDR (opline + 1) } else if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { | LOAD_IP_ADDR (opline + 1) } } if (!func) { |9: } return 1; } static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) { uint32_t arg_num = opline->op2.num; zend_jit_addr arg_addr; ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); if (!zend_jit_reuse_ip(Dst)) { return 0; } if (opline->opcode == ZEND_SEND_VAL_EX) { uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { /* Don't generate code that always throws exception */ return 0; } } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | mov r0, EX:RX->func | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz &exit_addr } else { | mov r0, EX:RX->func | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >1 |.cold_code |1: | SET_EX_OPLINE opline, r0 | jmp ->throw_cannot_pass_by_ref |.code } } arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 } } else { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 } return 1; } static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) { | mov FCARG1a, EX->call | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24) | jnz >1 |.cold_code |1: | SET_EX_OPLINE opline, r0 | EXT_CALL zend_handle_undef_args, r0 | test r0, r0 | jnz ->exception_handler | jmp >2 |.code |2: return 1; } static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) { zend_jit_addr op1_addr, arg_addr, ref_addr; op1_addr = OP1_ADDR(); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(Dst)) { return 0; } if (opline->op1_type == IS_VAR) { if (op1_info & MAY_BE_INDIRECT) { | LOAD_ZVAL_ADDR r0, op1_addr | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 | // ret = Z_INDIRECT_P(ret); | GET_Z_PTR r0, r0 |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); } } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL | jmp >2 |1: } op1_info &= ~MAY_BE_UNDEF; op1_info |= MAY_BE_NULL; } } else { ZEND_UNREACHABLE(); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { if (op1_info & MAY_BE_REF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 | GET_ZVAL_PTR r1, op1_addr | GC_ADDREF r1 | SET_ZVAL_PTR arg_addr, r1 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX | jmp >6 } |2: | // ZVAL_NEW_REF(arg, varptr); if (opline->op1_type == IS_VAR) { if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR r0, op1_addr } | mov aword T1, r0 // save } | EMALLOC sizeof(zend_reference), op_array, opline | mov dword [r0], 2 | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); if (opline->op1_type == IS_VAR) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); | mov r1, aword T1 // restore | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2 | SET_ZVAL_PTR val_addr, r0 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX } else { | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 | SET_ZVAL_PTR op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX } | SET_ZVAL_PTR arg_addr, r0 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX } |6: | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline |7: return 1; } static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) { uint32_t arg_num = opline->op2.num; zend_jit_addr arg_addr; ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || arg_num <= MAX_ARG_FLAG_NUM); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(Dst)) { return 0; } if (opline->opcode == ZEND_SEND_VAR_EX) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { return 0; } return 1; } } else { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); | mov r0, EX:RX->func | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >1 |.cold_code |1: if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { return 0; } | jmp >7 |.code } } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!(op1_info & MAY_BE_REF)) { /* Don't generate code that always throws exception */ return 0; } else { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | cmp cl, IS_REFERENCE | jne &exit_addr } } return 1; } } else { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); | mov r0, EX:RX->func | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >1 |.cold_code |1: mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (op1_info & MAY_BE_REF) { | cmp cl, IS_REFERENCE | je >7 } | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >7 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | jmp &exit_addr } else { | SET_EX_OPLINE opline, r0 | LOAD_ZVAL_ADDR FCARG1a, arg_addr | EXT_CALL zend_jit_only_vars_by_reference, r0 if (!zend_jit_check_exception(Dst)) { return 0; } | jmp >7 } |.code } } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { return 0; } return 1; } } else { | test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF | jnz >1 |.cold_code |1: if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { return 0; } | jmp >7 |.code } } if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 |.cold_code |1: } | SET_EX_OPLINE opline, r0 | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL | test r0, r0 | jz ->exception_handler if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | jmp >7 |.code } else { |7: return 1; } } if (opline->opcode == ZEND_SEND_VAR_NO_REF) { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 if (op1_info & MAY_BE_REF) { | cmp cl, IS_REFERENCE | je >7 } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | jmp &exit_addr } else { | SET_EX_OPLINE opline, r0 | LOAD_ZVAL_ADDR FCARG1a, arg_addr | EXT_CALL zend_jit_only_vars_by_reference, r0 if (!zend_jit_check_exception(Dst)) { return 0; } } } else { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2 | TRY_ADDREF op1_info, ah, r2 } else { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8); | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR FCARG1a, op1_addr | // ZVAL_COPY_VALUE(return_value, &ref->value); | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2 | GC_DELREF FCARG1a | je >1 | IF_NOT_REFCOUNTED ah, >2 | GC_ADDREF r2 | jmp >2 |1: | EFREE_REG_REFERENCE | jmp >2 |.code | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 |2: } } else { if (op1_addr != op1_def_addr) { if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { return 0; } if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { op1_addr= op1_def_addr; } } | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, r2 } } } |7: return 1; } static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) { uint32_t arg_num = opline->op2.num; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); || if (reuse_ip) { | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF || } else { | mov r0, EX->call | or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF || } } } else { if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); || if (reuse_ip) { | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF || } else { | mov r0, EX->call | and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF || } } } } else { // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); if (!zend_jit_reuse_ip(Dst)) { return 0; } | mov r0, EX:RX->func | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask | jnz >1 |.cold_code |1: | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF | jmp >1 |.code | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF |1: } return 1; } static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) { if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { if (jmp) { | jmp >7 } } else if (smart_branch_opcode == ZEND_JMPNZ) { | jmp =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | jmp =>target_label2 } else { ZEND_UNREACHABLE(); } } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE if (jmp) { | jmp >7 } } return 1; } static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) { if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { if (jmp) { | jmp >7 } } else if (smart_branch_opcode == ZEND_JMPZNZ) { | jmp =>target_label } else { ZEND_UNREACHABLE(); } } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE if (jmp) { | jmp >7 } } return 1; } static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t defined_label = (uint32_t)-1; uint32_t undefined_label = (uint32_t)-1; zval *zv = RT_CONSTANT(opline, opline->op1); zend_jit_addr res_addr = 0; if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { undefined_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { defined_label = target_label; } else if (smart_branch_opcode == ZEND_JMPZNZ) { undefined_label = target_label; defined_label = target_label2; } else { ZEND_UNREACHABLE(); } } | // if (CACHED_PTR(opline->extended_value)) { | mov r0, EX->run_time_cache | mov r0, aword [r0 + opline->extended_value] | test r0, r0 | jz >1 | test r0, 0x1 | jnz >4 |.cold_code |4: | MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, zend_constants, FCARG1a | shr r0, 1 | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jz &exit_addr } else { | jz >3 } } else if (undefined_label != (uint32_t)-1) { | jz =>undefined_label } else { | jz >3 } } else { | jz >2 } |1: | SET_EX_OPLINE opline, r0 | LOAD_ADDR FCARG1a, zv | EXT_CALL zend_jit_check_constant, r0 | test r0, r0 if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jz >3 } else { | jnz >3 } | jmp &exit_addr } else if (smart_branch_opcode) { if (undefined_label != (uint32_t)-1) { | jz =>undefined_label } else { | jz >3 } if (defined_label != (uint32_t)-1) { | jmp =>defined_label } else { | jmp >3 } } else { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); | jnz >1 |2: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE | jmp >3 } |.code if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jmp &exit_addr } } else if (defined_label != (uint32_t)-1) { | jmp =>defined_label } } else { |1: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } |3: return 1; } static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t mask; zend_jit_addr op1_addr = OP1_ADDR(); // TODO: support for is_resource() ??? ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 |.cold_code |1: } | SET_EX_OPLINE opline, r0 | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 zend_jit_check_exception_undef_result(Dst, opline); if (opline->extended_value & MAY_BE_NULL) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jmp &exit_addr } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { | jmp >7 } } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { return 0; } } else { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp &exit_addr } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { | jmp >7 } } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { return 0; } } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { |.code } } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { mask = opline->extended_value; if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jmp &exit_addr } } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { return 0; } } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp &exit_addr } } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { return 0; } } else { bool invert = 0; zend_uchar type; switch (mask) { case MAY_BE_NULL: type = IS_NULL; break; case MAY_BE_FALSE: type = IS_FALSE; break; case MAY_BE_TRUE: type = IS_TRUE; break; case MAY_BE_LONG: type = IS_LONG; break; case MAY_BE_DOUBLE: type = IS_DOUBLE; break; case MAY_BE_STRING: type = IS_STRING; break; case MAY_BE_ARRAY: type = IS_ARRAY; break; case MAY_BE_OBJECT: type = IS_OBJECT; break; case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; default: type = 0; } if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR r0, op1_addr | ZVAL_DEREF r0, op1_info } if (type == 0) { if (smart_branch_opcode && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { | IF_ZVAL_REFCOUNTED op1_addr, >1 |.cold_code |1: } | // if (!Z_DELREF_P(cv)) { | GET_ZVAL_PTR FCARG1a, op1_addr | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { | jnz >3 } if (op1_info & MAY_BE_REF) { | mov al, byte [r0 + 8] } else { | mov al, byte [FP + opline->op1.var + 8] } | mov byte T1, al // save | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op1_info, opline | mov cl, byte T1 // restore |jmp >2 } if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (!RC_MAY_BE_1(op1_info)) { | jmp >3 } |.code } |3: if (op1_info & MAY_BE_REF) { | mov cl, byte [r0 + 8] } else { | mov cl, byte [FP + opline->op1.var + 8] } |2: } else { if (op1_info & MAY_BE_REF) { | mov cl, byte [r0 + 8] } else { | mov cl, byte [FP + opline->op1.var + 8] } } | mov eax, 1 | shl eax, cl | test eax, mask if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jne &exit_addr } else { | je &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | je =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | jne =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | je =>target_label | jmp =>target_label2 } else { ZEND_UNREACHABLE(); } } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); | setne al | movzx eax, al | add eax, 2 | SET_ZVAL_TYPE_INFO res_addr, eax | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline } } else { if (smart_branch_opcode && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { | IF_ZVAL_REFCOUNTED op1_addr, >1 |.cold_code |1: } | // if (!Z_DELREF_P(cv)) { | GET_ZVAL_PTR FCARG1a, op1_addr | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { | jnz >3 } if (op1_info & MAY_BE_REF) { | mov al, byte [r0 + 8] } else { | mov al, byte [FP + opline->op1.var + 8] } | mov byte T1, al // save | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op1_info, opline | mov cl, byte T1 // restore |jmp >2 } if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (!RC_MAY_BE_1(op1_info)) { | jmp >3 } |.code } |3: if (op1_info & MAY_BE_REF) { | mov cl, byte [r0 + 8] } else { | mov cl, byte [FP + opline->op1.var + 8] } |2: | cmp cl, type } else { if (op1_info & MAY_BE_REF) { | cmp byte [r0 + 8], type } else { | cmp byte [FP + opline->op1.var + 8], type } } if (exit_addr) { if (invert) { if (smart_branch_opcode == ZEND_JMPNZ) { | jne &exit_addr } else { | je &exit_addr } } else { if (smart_branch_opcode == ZEND_JMPNZ) { | je &exit_addr } else { | jne &exit_addr } } } else if (smart_branch_opcode) { if (invert) { if (smart_branch_opcode == ZEND_JMPZ) { | je =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | jne =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | je =>target_label | jmp =>target_label2 } else { ZEND_UNREACHABLE(); } } else { if (smart_branch_opcode == ZEND_JMPZ) { | jne =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | je =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | jne =>target_label | jmp =>target_label2 } else { ZEND_UNREACHABLE(); } } } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (invert) { | setne al } else { | sete al } | movzx eax, al | add eax, 2 | SET_ZVAL_TYPE_INFO res_addr, eax | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline } } } } |7: return 1; } static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) { uint32_t j, info; if (ssa->vars && ssa->var_info) { info = ssa->var_info[var].type; for (j = op_array->last_var; j < ssa->vars_count; j++) { if (ssa->vars[j].var == var) { info |= ssa->var_info[j].type; } } } else { info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } #ifdef ZEND_JIT_USE_RC_INFERENCE /* Refcount may be increased by RETURN opcode */ if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { for (j = 0; j < ssa->cfg.blocks_count; j++) { if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && ssa->cfg.blocks[j].len > 0) { const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; if (opline->opcode == ZEND_RETURN) { if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) { info |= MAY_BE_RCN; break; } } } } } #endif return info; } static int zend_jit_leave_frame(dasm_State **Dst) { | // EG(current_execute_data) = EX(prev_execute_data); | mov r0, EX->prev_execute_data | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2 return 1; } static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) { if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = EX_NUM_TO_VAR(var); | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL } return 1; } static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) { if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline } return 1; } static int zend_jit_leave_func(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, uint32_t op1_info, bool left_frame, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info, int indirect_var_access, int may_throw) { bool may_be_top_frame = JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); bool may_need_call_helper = indirect_var_access || /* may have symbol table */ !op_array->function_name || /* may have symbol table */ may_be_top_frame || (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ bool may_need_release_this = !(op_array->fn_flags & ZEND_ACC_CLOSURE) && op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC) && (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !TRACE_FRAME_NO_NEED_REKEASE_THIS(JIT_G(current_frame))); if (may_need_call_helper || may_need_release_this) { | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] } if (may_need_call_helper) { if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(Dst)) { return 0; } } /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) if (trace && trace->op != ZEND_JIT_TRACE_END) { | jnz >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG2a, FP } | EXT_CALL zend_jit_leave_func_helper, r0 if (may_be_top_frame) { // TODO: try to avoid this check ??? if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { #if 0 /* this check should be handled by the following OPLINE guard */ | cmp IP, zend_jit_halt_op | je ->trace_halt #endif } else if (GCC_GLOBAL_REGS) { | test IP, IP | je ->trace_halt } else { | test eax, eax | jl ->trace_halt } } if (!GCC_GLOBAL_REGS) { | // execute_data = EG(current_execute_data) | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 } | jmp >8 |.code } else { | jnz ->leave_function_handler } } if (op_array->fn_flags & ZEND_ACC_CLOSURE) { if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(Dst)) { return 0; } } | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); | mov FCARG1a, EX->func | sub FCARG1a, sizeof(zend_object) | OBJ_RELEASE ZREG_FCARG1a, >4 |4: } else if (may_need_release_this) { if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(Dst)) { return 0; } } | // if (call_info & ZEND_CALL_RELEASE_THIS) | test FCARG1d, ZEND_CALL_RELEASE_THIS | je >4 | // zend_object *object = Z_OBJ(execute_data->This); | mov FCARG1a, EX->This.value.obj | // OBJ_RELEASE(object); | OBJ_RELEASE ZREG_FCARG1a, >4 |4: // TODO: avoid EG(excption) check for $this->foo() calls may_throw = 1; } | // EG(vm_stack_top) = (zval*)execute_data; | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0 | // execute_data = EX(prev_execute_data); | mov FP, EX->prev_execute_data if (!left_frame) { | // EG(current_execute_data) = execute_data; | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 } |9: if (trace) { if (trace->op != ZEND_JIT_TRACE_END && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { zend_jit_reset_last_valid_opline(); } else { | LOAD_IP | ADD_IP sizeof(zend_op) } |8: if (trace->op == ZEND_JIT_TRACE_BACK && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { const zend_op *next_opline = trace->opline; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & MAY_BE_RC1) && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { /* exception might be thrown during destruction of unused return value */ | // if (EG(exception)) | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->leave_throw_handler } do { trace++; } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); next_opline = trace->opline; ZEND_ASSERT(next_opline != NULL); if (trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { trace_info->flags |= ZEND_JIT_TRACE_LOOP; | CMP_IP next_opline | je =>0 // LOOP #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | JMP_IP #else | jmp ->trace_escape #endif } else { | CMP_IP next_opline | jne ->trace_escape } zend_jit_set_last_valid_opline(trace->opline); return 1; } else if (may_throw || (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & MAY_BE_RC1) && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { | // if (EG(exception)) | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | jne ->leave_throw_handler } return 1; } else { | // if (EG(exception)) | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 | LOAD_IP | jne ->leave_throw_handler | // opline = EX(opline) + 1 | ADD_IP sizeof(zend_op) } if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD #ifdef CONTEXT_THREADED_JIT | push aword [IP] | ret #else | JMP_IP #endif } else if (GCC_GLOBAL_REGS) { | add r4, SPAD // stack alignment #ifdef CONTEXT_THREADED_JIT | push aword [IP] | ret #else | JMP_IP #endif } else { #ifdef CONTEXT_THREADED_JIT ZEND_UNREACHABLE(); // TODO: context threading can't work without GLOBAL REGS because we have to change // the value of execute_data in execute_ex() | mov FCARG1a, FP | mov r0, aword [FP] | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | push aword [r0] | ret #else | mov FP, aword T2 // restore FP | mov RX, aword T3 // restore IP | add r4, NR_SPAD // stack alignment | mov r0, 2 // ZEND_VM_LEAVE | ret #endif } return 1; } static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) { zend_jit_addr ret_addr; int8_t return_value_used; ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { return_value_used = 1; } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { return_value_used = 0; } else { return_value_used = -1; } } else { return_value_used = -1; } if (ZEND_OBSERVER_ENABLED) { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { return 0; } op1_addr = dst; } | LOAD_ZVAL_ADDR FCARG2a, op1_addr | mov FCARG1a, FP | SET_EX_OPLINE opline, r0 | EXT_CALL zend_observer_fcall_end, r0 } // if (!EX(return_value)) if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { if (return_value_used != 0) { | mov r2, EX->return_value } if (return_value_used == -1) { | test r2, r2 } ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); } else { if (return_value_used != 0) { | mov r1, EX->return_value } if (return_value_used == -1) { | test r1, r1 } ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (return_value_used == -1) { | jz >1 |.cold_code |1: } if (return_value_used != 1) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (jit_return_label >= 0) { | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label } else { | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 } } | GET_ZVAL_PTR FCARG1a, op1_addr | GC_DELREF FCARG1a if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { if (jit_return_label >= 0) { | jnz =>jit_return_label } else { | jnz >9 } } | //SAVE_OPLINE() | ZVAL_DTOR_FUNC op1_info, opline | //????mov r1, EX->return_value // reload ??? } if (return_value_used == -1) { if (jit_return_label >= 0) { | jmp =>jit_return_label } else { | jmp >9 } |.code } } } else if (return_value_used == -1) { if (jit_return_label >= 0) { | jz =>jit_return_label } else { | jz >9 } } if (return_value_used == 0) { |9: return 1; } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 } } else if (opline->op1_type == IS_TMP_VAR) { | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR r0, op1_addr | ZVAL_DEREF r0, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); } | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || !op_array->function_name) { | TRY_ADDREF op1_info, ah, r2 } else if (return_value_used != 1) { | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL } } } else { if (op1_info & MAY_BE_REF) { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR r0, op1_addr | // ZVAL_COPY_VALUE(return_value, &ref->value); | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2 | GC_DELREF r0 | je >2 | // if (IS_REFCOUNTED()) if (jit_return_label >= 0) { | IF_NOT_REFCOUNTED dh, =>jit_return_label } else { | IF_NOT_REFCOUNTED dh, >9 } | // ADDREF | GET_ZVAL_PTR r2, ret_addr // reload | GC_ADDREF r2 if (jit_return_label >= 0) { | jmp =>jit_return_label } else { | jmp >9 } |2: | EFREE_REFERENCE r0 if (jit_return_label >= 0) { | jmp =>jit_return_label } else { | jmp >9 } |.code } | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 } |9: return 1; } static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) { ZEND_ASSERT(type_reg == ZREG_R2); |.if not(X64) || if (Z_REG(val_addr) == ZREG_R1) { | GET_ZVAL_W2 r0, val_addr || } |.endif | GET_ZVAL_PTR r1, val_addr |.if not(X64) || if (Z_REG(val_addr) != ZREG_R1) { | GET_ZVAL_W2 r0, val_addr || } |.endif | IF_NOT_REFCOUNTED dh, >2 | IF_NOT_TYPE dl, IS_REFERENCE, >1 | GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val) |.if not(X64) | GET_Z_W2 r0, r1+offsetof(zend_reference, val) |.endif | GET_Z_PTR r1, r1+offsetof(zend_reference, val) | IF_NOT_REFCOUNTED dh, >2 |1: | GC_ADDREF r1 |2: | SET_ZVAL_PTR res_addr, r1 |.if not(X64) | SET_ZVAL_W2 res_addr, r0 |.endif | SET_ZVAL_TYPE_INFO res_addr, edx return 1; } static bool zend_jit_may_avoid_refcounting(const zend_op *opline) { switch (opline->opcode) { case ZEND_FETCH_OBJ_FUNC_ARG: if (!JIT_G(current_frame) || !JIT_G(current_frame)->call->func || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { return 0; } /* break missing intentionally */ case ZEND_FETCH_OBJ_R: case ZEND_FETCH_OBJ_IS: if (opline->op2_type == IS_CONST && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') { return 1; } break; case ZEND_FETCH_DIM_FUNC_ARG: if (!JIT_G(current_frame) || !JIT_G(current_frame)->call->func || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { return 0; } /* break missing intentionally */ case ZEND_FETCH_DIM_R: case ZEND_FETCH_DIM_IS: return 1; case ZEND_ISSET_ISEMPTY_DIM_OBJ: if (!(opline->extended_value & ZEND_ISEMPTY)) { return 1; } break; } return 0; } static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_avoid_refcounting, uint32_t op2_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) { zend_jit_addr orig_op1_addr, op2_addr; const void *exit_addr = NULL; const void *not_found_exit_addr = NULL; const void *res_exit_addr = NULL; bool result_avoid_refcounting = 0; uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; orig_op1_addr = OP1_ADDR(); op2_addr = OP2_ADDR(); if (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { uint32_t flags = 0; uint32_t old_op1_info = 0; uint32_t old_info; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; int32_t exit_point; if (opline->opcode != ZEND_FETCH_LIST_R && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !op1_avoid_refcounting) { flags |= ZEND_JIT_EXIT_FREE_OP1; } if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { flags |= ZEND_JIT_EXIT_FREE_OP2; } if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) && !(flags & ZEND_JIT_EXIT_FREE_OP1) && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) && (ssa_op+1)->op1_use == ssa_op->result_def && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) && zend_jit_may_avoid_refcounting(opline+1)) { result_avoid_refcounting = 1; ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; } if (op1_avoid_refcounting) { old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!res_exit_addr) { return 0; } res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; } if (opline->opcode == ZEND_FETCH_DIM_IS && !(res_info & MAY_BE_NULL)) { old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { return 0; } } if (op1_avoid_refcounting) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); } } if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } } | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { return 0; } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 } } | SET_EX_OPLINE opline, r0 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr if (opline->opcode != ZEND_FETCH_DIM_IS) { if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0 } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 } | SET_ZVAL_PTR res_addr, r0 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 |.if not(X64) | add r4, 12 |.endif } if ((op1_info & MAY_BE_ARRAY) || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { | jmp >9 // END } |6: } if (op1_info & MAY_BE_OBJECT) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { if (exit_addr) { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 } } | SET_EX_OPLINE opline, r0 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif if (opline->opcode != ZEND_FETCH_DIM_IS) { | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 } else { | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 } |.if not(X64) | add r4, 12 |.endif if ((op1_info & MAY_BE_ARRAY) || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { | jmp >9 // END } |6: } if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { | SET_EX_OPLINE opline, r0 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } if (op2_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } } if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_STRING)))) { if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr } else { | SET_EX_OPLINE opline, r0 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } } | EXT_CALL zend_jit_invalid_array_access, r0 } | SET_ZVAL_TYPE_INFO res_addr, IS_NULL if (op1_info & MAY_BE_ARRAY) { | jmp >9 // END } } if (op1_info & MAY_BE_ARRAY) { |.code } } if (op1_info & MAY_BE_ARRAY) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); |8: if (res_exit_addr) { zend_uchar type = concrete_type(res_info); if (op1_info & MAY_BE_ARRAY_OF_REF) { | ZVAL_DEREF r0, MAY_BE_REF } if (type < IS_STRING) { | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr } else { | GET_ZVAL_TYPE_INFO edx, val_addr | IF_NOT_TYPE dl, type, &res_exit_addr } | // ZVAL_COPY |7: | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (type < IS_STRING) { if (Z_REG(res_addr) != ZREG_FP || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { | SET_ZVAL_TYPE_INFO res_addr, type } } else { | SET_ZVAL_TYPE_INFO res_addr, edx if (!result_avoid_refcounting) { | TRY_ADDREF res_info, dh, r1 } } } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } } else if (op1_info & MAY_BE_ARRAY_OF_REF) { | // ZVAL_COPY_DEREF | GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) { return 0; } } else { | // ZVAL_COPY | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2 | TRY_ADDREF res_info, ch, r2 } } |9: // END #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { /* Magic offsetGet() may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_fetch_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { zend_jit_addr op2_addr; op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 | GET_Z_PTR FCARG2a, FCARG1a | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] | jmp >3 |.cold_code |2: | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 | test r0, r0 | mov FCARG1a, r0 | jne >1 | jmp ->exception_handler_undef |.code |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_FALSE | jg >7 } if ((op1_info & MAY_BE_UNDEF) && opline->opcode == ZEND_FETCH_DIM_RW) { if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 } | SET_EX_OPLINE opline, r0 | mov FCARG1a, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | mov T1, Ra(Z_REG(op1_addr)) // save } | EXT_CALL _zend_new_array_0, r0 if (Z_REG(op1_addr) != ZREG_FP) { | mov Ra(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX | mov FCARG1a, r0 if (op1_info & MAY_BE_ARRAY) { | jmp >1 |.code |1: } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |6: if (opline->op2_type == IS_UNUSED) { | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval | EXT_CALL zend_hash_next_index_insert, r0 | // if (UNEXPECTED(!var_ptr)) { | test r0, r0 | jz >1 |.cold_code |1: | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); | CANNOT_ADD_ELEMENT opline | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | jmp >8 |.code | SET_ZVAL_PTR res_addr, r0 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT } else { uint32_t type; switch (opline->opcode) { case ZEND_FETCH_DIM_W: case ZEND_FETCH_LIST_W: type = BP_VAR_W; break; case ZEND_FETCH_DIM_RW: type = BP_VAR_RW; break; case ZEND_FETCH_DIM_UNSET: type = BP_VAR_UNSET; break; default: ZEND_UNREACHABLE(); } if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, NULL, NULL, NULL)) { return 0; } |8: | SET_ZVAL_PTR res_addr, r0 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { |.cold_code |9: | SET_ZVAL_TYPE_INFO res_addr, IS_NULL | jmp >8 |.code } } } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |.cold_code |7: } | SET_EX_OPLINE opline, r0 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_UNUSED) { | xor FCARG2a, FCARG2a } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif switch (opline->opcode) { case ZEND_FETCH_DIM_W: case ZEND_FETCH_LIST_W: | EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0 break; case ZEND_FETCH_DIM_RW: | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0 break; // case ZEND_FETCH_DIM_UNSET: // | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0 // break; default: ZEND_UNREACHABLE(); } |.if not(X64) | add r4, 12 |.endif if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { | jmp >8 // END |.code } } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { /* ASSIGN_DIM may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif |8: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_avoid_refcounting, uint32_t op2_info, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_jit_addr op2_addr, res_addr; // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); op2_addr = OP2_ADDR(); res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | ZVAL_DEREF FCARG1a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_ARRAY) { const void *found_exit_addr = NULL; const void *not_found_exit_addr = NULL; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr if (exit_addr && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) && !may_throw && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { if (smart_branch_opcode == ZEND_JMPNZ) { found_exit_addr = exit_addr; } else { not_found_exit_addr = exit_addr; } } if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, found_exit_addr, not_found_exit_addr, NULL)) { return 0; } if (found_exit_addr) { |9: return 1; } else if (not_found_exit_addr) { |8: return 1; } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { | SET_EX_OPLINE opline, r0 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2a, op2_addr } | EXT_CALL zend_jit_isset_dim_helper, r0 | test r0, r0 | jz >9 if (op1_info & MAY_BE_ARRAY) { | jmp >8 |.code } } else { if (op2_info & MAY_BE_UNDEF) { if (op2_info & MAY_BE_ANY) { | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 } | mov FCARG1d, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: } if (op1_info & MAY_BE_ARRAY) { | jmp >9 |.code } } } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { /* Magic offsetExists() may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { |8: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline if (!op1_avoid_refcounting) { | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline } if (may_throw) { if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } if (!(opline->extended_value & ZEND_ISEMPTY)) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jmp &exit_addr } else { | jmp >8 } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp =>target_label2 } else if (smart_branch_opcode == ZEND_JMPNZ) { | jmp =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | jmp =>target_label2 } else { ZEND_UNREACHABLE(); } } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE | jmp >8 } } else { | //???? | int3 } } |9: // not found | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline if (!op1_avoid_refcounting) { | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline } if (may_throw) { if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } if (!(opline->extended_value & ZEND_ISEMPTY)) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | jmp =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { } else if (smart_branch_opcode == ZEND_JMPZNZ) { | jmp =>target_label } else { ZEND_UNREACHABLE(); } } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } } else { | //???? | int3 } |8: return 1; } static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { zend_jit_addr op1_addr = OP1_ADDR(); zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; | mov r0, EX->run_time_cache | mov r0, aword [r0 + opline->extended_value] | sub r0, 1 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) | MEM_OP2_2_ZTS mov, ecx, dword, executor_globals, symbol_table.nNumUsed, r1 |.if X64 | shl r1, 5 |.else | imul r1, sizeof(Bucket) |.endif | cmp r0, r1 | jae >9 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); | MEM_OP2_2_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >9 | // (EXPECTED(p->key == varname)) | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, key)], varname, r1 | jne >9 | GET_Z_PTR r0, r0 | GC_ADDREF r0 |1: if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) | IF_ZVAL_REFCOUNTED op1_addr, >2 |.cold_code |2: } | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); | GET_ZVAL_PTR FCARG1a, op1_addr | // ZVAL_REF(variable_ptr, ref) | SET_ZVAL_PTR op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX | // if (GC_DELREF(garbage) == 0) | GC_DELREF FCARG1a if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { | jnz >3 } else { | jnz >5 } | ZVAL_DTOR_FUNC op1_info, opline | jmp >5 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { |3: | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) | IF_GC_MAY_NOT_LEAK FCARG1a, >5 | EXT_CALL gc_possible_root, r1 | jmp >5 } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { |.code } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // ZVAL_REF(variable_ptr, ref) | SET_ZVAL_PTR op1_addr, r0 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX } |5: //END of handler |.cold_code |9: | LOAD_ADDR FCARG1a, (ptrdiff_t)varname | mov FCARG2a, EX->run_time_cache if (opline->extended_value) { | add FCARG2a, opline->extended_value } | EXT_CALL zend_jit_fetch_global_helper, r0 | jmp <1 |.code return 1; } static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); bool in_cold = 0; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1a : ZREG_R0; if (ZEND_ARG_SEND_MODE(arg_info)) { if (opline->opcode == ZEND_RECV_INIT) { | LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr | ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); } else { | GET_ZVAL_PTR Ra(tmp_reg), res_addr res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); } } if (type_mask != 0) { if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); | IF_NOT_ZVAL_TYPE res_addr, type_code, >1 } else { | mov edx, 1 | mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)] | shl edx, cl | test edx, type_mask | je >1 } |.cold_code |1: in_cold = 1; } if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, res_addr } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | SET_EX_OPLINE opline, r0 } else { | ADDR_OP2_2 mov, aword EX->opline, opline, r0 } | LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info | EXT_CALL zend_jit_verify_arg_slow, r0 if (check_exception) { | test al, al if (in_cold) { | jnz >1 | jmp ->exception_handler |.code |1: } else { | jz ->exception_handler } } else if (in_cold) { | jmp >1 |.code |1: } return 1; } static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) { uint32_t arg_num = opline->op1.num; zend_arg_info *arg_info = NULL; if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { if (EXPECTED(arg_num <= op_array->num_args)) { arg_info = &op_array->arg_info[arg_num-1]; } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { arg_info = &op_array->arg_info[op_array->num_args]; } if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { arg_info = NULL; } } if (arg_info || (opline+1)->opcode != ZEND_RECV) { | cmp dword EX->This.u2.num_args, arg_num if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | jb &exit_addr } else { | jb >1 |.cold_code |1: if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | SET_EX_OPLINE opline, r0 } else { | ADDR_OP2_2 mov, aword EX->opline, opline, r0 } | mov FCARG1a, FP | EXT_CALL zend_missing_arg_error, r0 | jmp ->exception_handler |.code } } if (arg_info) { if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { return 0; } } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { | LOAD_IP_ADDR (opline + 1) zend_jit_set_last_valid_opline(opline + 1); } } return 1; } static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) { uint32_t arg_num = opline->op1.num; zval *zv = RT_CONSTANT(opline, opline->op2); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | cmp dword EX->This.u2.num_args, arg_num | jae >5 } | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 } if (Z_CONSTANT_P(zv)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | SET_EX_OPLINE opline, r0 } else { | ADDR_OP2_2 mov, aword EX->opline, opline, r0 } | LOAD_ZVAL_ADDR FCARG1a, res_addr | mov r0, EX->func | mov FCARG2a, [r0 + offsetof(zend_op_array, scope)] | .if X64 | EXT_CALL zval_update_constant_ex, r0 | .else ||#if (PHP_VERSION_ID < 80100) && (SIZEOF_SIZE_T == 4) | EXT_CALL zval_jit_update_constant_ex, r0 ||#else | EXT_CALL zval_update_constant_ex, r0 ||#endif | .endif | test al, al | jnz >1 |.cold_code |1: | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF | jmp ->exception_handler |.code } |5: if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { do { zend_arg_info *arg_info; if (arg_num <= op_array->num_args) { arg_info = &op_array->arg_info[arg_num-1]; } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { arg_info = &op_array->arg_info[op_array->num_args]; } else { break; } if (!ZEND_TYPE_IS_SET(arg_info->type)) { break; } if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { return 0; } } while (0); } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { if (is_last) { | LOAD_IP_ADDR (opline + 1) zend_jit_set_last_valid_opline(opline + 1); } } return 1; } static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) { zend_property_info *info = NULL; if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) || !ce || !(ce->ce_flags & ZEND_ACC_LINKED) || (ce->ce_flags & ZEND_ACC_TRAIT) || ce->create_object) { return NULL; } if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { if (ce->info.user.filename != filename) { /* class declaration might be changed independently */ return NULL; } if (ce->parent) { zend_class_entry *parent = ce->parent; do { if (parent->type == ZEND_INTERNAL_CLASS) { break; } else if (parent->info.user.filename != filename) { /* some of parents class declarations might be changed independently */ /* TODO: this check may be not enough, because even * in the same it's possible to conditionally define * few classes with the same name, and "parent" may * change from request to request. */ return NULL; } parent = parent->parent; } while (parent); } } info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); if (info == NULL || !IS_VALID_PROPERTY_OFFSET(info->offset) || (info->flags & ZEND_ACC_STATIC)) { return NULL; } if (!(info->flags & ZEND_ACC_PUBLIC) && (!on_this || info->ce != ce)) { return NULL; } return info; } static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) { zend_property_info *info; if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { return 1; } if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { if (ce->info.user.filename != filename) { /* class declaration might be changed independently */ return 1; } } info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); if (info == NULL || !IS_VALID_PROPERTY_OFFSET(info->offset) || (info->flags & ZEND_ACC_STATIC)) { return 1; } if (!(info->flags & ZEND_ACC_PUBLIC) && (!on_this || info->ce != ce)) { return 1; } return 0; } static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } |.if X64 || if (!IS_SIGNED_32BIT(ce)) { | mov64 r0, ((ptrdiff_t)ce) | cmp aword [FCARG1a + offsetof(zend_object, ce)], r0 || } else { | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce || } |.else | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce |.endif | jne &exit_addr return 1; } static int zend_jit_fetch_obj(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool use_this, bool op1_avoid_refcounting, zend_class_entry *trace_ce, int may_throw) { zval *member; zend_property_info *prop_info; bool may_be_dynamic = 1; zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; uint32_t res_info = RES_INFO(); ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { | GET_ZVAL_PTR FCARG1a, this_addr } else { if (opline->op1_type == IS_VAR && opline->opcode == ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 | GET_Z_PTR FCARG1a, FCARG1a |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_REF) { if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | ZVAL_DEREF FCARG1a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 } } | GET_ZVAL_PTR FCARG1a, op1_addr } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (!zend_jit_class_guard(Dst, opline, trace_ce)) { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } } } } if (!prop_info) { | mov r0, EX->run_time_cache | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >5 | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (may_be_dynamic) { | test r0, r0 if (opline->opcode == ZEND_FETCH_OBJ_W) { | jl >5 } else { | jl >8 // dynamic property } } | mov edx, dword [FCARG1a + r0 + 8] | IF_UNDEF dl, >5 | add FCARG1a, r0 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); if (opline->opcode == ZEND_FETCH_OBJ_W && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; | mov r0, EX->run_time_cache | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] | test FCARG2a, FCARG2a | jnz >1 |.cold_code |1: if (flags == ZEND_FETCH_DIM_WRITE) { | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_check_array_promotion, r0 | jmp >9 } else if (flags == ZEND_FETCH_REF) { |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_create_typed_ref, r0 |.if not(X64) | add r4, 12 |.endif | jmp >9 } else { ZEND_UNREACHABLE(); } |.code } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); | mov edx, dword [FCARG1a + prop_info->offset + 8] if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_UNDEF dl, &exit_addr } } else { | IF_UNDEF dl, >5 } if (opline->opcode == ZEND_FETCH_OBJ_W && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) && ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; if (flags == ZEND_FETCH_DIM_WRITE) { if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { | cmp dl, IS_FALSE | jle >1 |.cold_code |1: if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, prop_addr } | LOAD_ADDR FCARG2a, prop_info | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_check_array_promotion, r0 | jmp >9 |.code } } else if (flags == ZEND_FETCH_REF) { | IF_TYPE dl, IS_REFERENCE, >1 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2a, prop_info } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] | mov FCARG2a, aword[r0 + prop_info_offset] } if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, prop_addr } |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif | EXT_CALL zend_jit_create_typed_ref, r0 |.if not(X64) | add r4, 12 |.endif | jmp >9 |1: } else { ZEND_UNREACHABLE(); } } } if (op1_avoid_refcounting) { SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } if (opline->opcode == ZEND_FETCH_OBJ_W) { if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, prop_addr } | SET_ZVAL_PTR res_addr, FCARG1a | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { ssa->var_info[ssa_op->result_def].indirect_reference = 1; } } else { bool result_avoid_refcounting = 0; if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { uint32_t flags = 0; uint32_t old_info; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; int32_t exit_point; const void *exit_addr; zend_uchar type; zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this && !op1_avoid_refcounting) { flags = ZEND_JIT_EXIT_FREE_OP1; } | LOAD_ZVAL_ADDR r0, prop_addr if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) && !(flags & ZEND_JIT_EXIT_FREE_OP1) && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) && (ssa_op+1)->op1_use == ssa_op->result_def && zend_jit_may_avoid_refcounting(opline+1)) { result_avoid_refcounting = 1; ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; } old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; type = concrete_type(res_info); | // ZVAL_DEREF() | IF_NOT_TYPE dl, IS_REFERENCE, >1 | GET_Z_PTR r0, r0 | add r0, offsetof(zend_reference, val) if (type < IS_STRING) { |1: | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr } else { | GET_ZVAL_TYPE_INFO edx, val_addr |1: | IF_NOT_TYPE dl, type, &exit_addr } | // ZVAL_COPY | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 if (type < IS_STRING) { if (Z_REG(res_addr) != ZREG_FP || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { | SET_ZVAL_TYPE_INFO res_addr, type } } else { | SET_ZVAL_TYPE_INFO res_addr, edx if (!result_avoid_refcounting) { | TRY_ADDREF res_info, dh, r1 } } } else { if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) { return 0; } } } |.cold_code if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { |5: | SET_EX_OPLINE opline, r0 if (opline->opcode == ZEND_FETCH_OBJ_W) { | EXT_CALL zend_jit_fetch_obj_w_slow, r0 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_r_slow, r0 } else { | EXT_CALL zend_jit_fetch_obj_is_slow, r0 } | jmp >9 } if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { |7: if (opline->opcode != ZEND_FETCH_OBJ_IS) { | SET_EX_OPLINE opline, r0 if (opline->opcode != ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_UNDEF)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); if (op1_info & MAY_BE_ANY) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 } | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 |1: | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr } else if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | LOAD_ADDR FCARG2a, Z_STRVAL_P(member) if (opline->opcode == ZEND_FETCH_OBJ_W) { | EXT_CALL zend_jit_invalid_property_write, r0 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR } else { | EXT_CALL zend_jit_invalid_property_read, r0 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL } | jmp >9 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_NULL | jmp >9 } } if (!prop_info && may_be_dynamic && opline->opcode != ZEND_FETCH_OBJ_W) { |8: | mov FCARG2a, r0 | SET_EX_OPLINE opline, r0 if (opline->opcode != ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 } else { | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 } | jmp >9 } |.code; |9: // END if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { if (opline->op1_type == IS_VAR && opline->opcode == ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_RC1)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1 | GET_ZVAL_PTR FCARG1a, orig_op1_addr | GC_DELREF FCARG1a | jnz >1 | SET_EX_OPLINE opline, r0 | EXT_CALL zend_jit_extract_helper, r0 |1: } else if (!op1_avoid_refcounting) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline } } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info && opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR) { may_throw = 0; } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_incdec_obj(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool use_this, zend_class_entry *trace_ce, int may_throw) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr res_addr = 0; zend_jit_addr prop_addr; bool needs_slow_path = 0; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); if (opline->result_type != IS_UNUSED) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { | GET_ZVAL_PTR FCARG1a, this_addr } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 | GET_Z_PTR FCARG1a, FCARG1a |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_REF) { if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | ZVAL_DEREF FCARG1a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 |.cold_code |1: | SET_EX_OPLINE opline, r0 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | LOAD_ADDR FCARG2a, ZSTR_VAL(name) | EXT_CALL zend_jit_invalid_property_incdec, r0 | jmp ->exception_handler |.code } } | GET_ZVAL_PTR FCARG1a, op1_addr } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (!zend_jit_class_guard(Dst, opline, trace_ce)) { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } if (!prop_info) { needs_slow_path = 1; | mov r0, EX->run_time_cache | mov r2, aword [r0 + opline->extended_value] | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >7 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0 | jnz >7 } | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] | test r0, r0 | jl >7 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 | add FCARG1a, r0 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr } else { | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 needs_slow_path = 1; } if (ZEND_TYPE_IS_SET(prop_info->type)) { | SET_EX_OPLINE opline, r0 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2a, prop_info } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] | mov FCARG2a, aword[r0 + prop_info_offset] } | LOAD_ZVAL_ADDR FCARG1a, prop_addr if (opline->result_type == IS_UNUSED) { switch (opline->opcode) { case ZEND_PRE_INC_OBJ: case ZEND_POST_INC_OBJ: | EXT_CALL zend_jit_inc_typed_prop, r0 break; case ZEND_PRE_DEC_OBJ: case ZEND_POST_DEC_OBJ: | EXT_CALL zend_jit_dec_typed_prop, r0 break; default: ZEND_UNREACHABLE(); } } else { |.if X64 | LOAD_ZVAL_ADDR CARG3, res_addr |.else | sub r4, 12 | PUSH_ZVAL_ADDR res_addr, r0 |.endif switch (opline->opcode) { case ZEND_PRE_INC_OBJ: | EXT_CALL zend_jit_pre_inc_typed_prop, r0 break; case ZEND_PRE_DEC_OBJ: | EXT_CALL zend_jit_pre_dec_typed_prop, r0 break; case ZEND_POST_INC_OBJ: | EXT_CALL zend_jit_post_inc_typed_prop, r0 break; case ZEND_POST_DEC_OBJ: | EXT_CALL zend_jit_post_dec_typed_prop, r0 break; default: ZEND_UNREACHABLE(); } |.if not(X64) | add r4, 12 |.endif } } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { zend_jit_addr var_addr = prop_addr; var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, prop_addr } | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 | GET_ZVAL_PTR FCARG1a, var_addr | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 | jnz >1 | lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)] |.cold_code |1: if (opline) { | SET_EX_OPLINE opline, r0 } if (opline->result_type == IS_UNUSED) { | xor FCARG2a, FCARG2a } else { | LOAD_ZVAL_ADDR FCARG2a, res_addr } switch (opline->opcode) { case ZEND_PRE_INC_OBJ: | EXT_CALL zend_jit_pre_inc_typed_ref, r0 break; case ZEND_PRE_DEC_OBJ: | EXT_CALL zend_jit_pre_dec_typed_ref, r0 break; case ZEND_POST_INC_OBJ: | EXT_CALL zend_jit_post_inc_typed_ref, r0 break; case ZEND_POST_DEC_OBJ: | EXT_CALL zend_jit_post_dec_typed_ref, r0 break; default: ZEND_UNREACHABLE(); } | jmp >9 |.code |2: | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { if (opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2 } } if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { | LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1) } else { | LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1) } | jo >3 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { if (opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2 } } |.cold_code |2: if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2 | TRY_ADDREF MAY_BE_ANY, ah, r2 } if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { | LOAD_ZVAL_ADDR FCARG2a, res_addr | EXT_CALL zend_jit_pre_inc, r0 } else { | EXT_CALL increment_function, r0 } } else { if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { | LOAD_ZVAL_ADDR FCARG2a, res_addr | EXT_CALL zend_jit_pre_dec, r0 } else { | EXT_CALL decrement_function, r0 } } | jmp >4 |3: if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { |.if X64 | mov64 rax, 0x43e0000000000000 | SET_ZVAL_LVAL var_addr, rax if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { | SET_ZVAL_LVAL res_addr, rax } |.else | SET_ZVAL_LVAL var_addr, 0 | SET_ZVAL_W2 var_addr, 0x41e00000 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { | SET_ZVAL_LVAL res_addr, 0 | SET_ZVAL_W2 res_addr, 0x41e00000 } |.endif } else { |.if X64 | mov64 rax, 0xc3e0000000000000 | SET_ZVAL_LVAL var_addr, rax if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { | SET_ZVAL_LVAL res_addr, rax } |.else | SET_ZVAL_LVAL var_addr, 0x00200000 | SET_ZVAL_W2 var_addr, 0xc1e00000 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { | SET_ZVAL_LVAL res_addr, 0x00200000 | SET_ZVAL_W2 res_addr, 0xc1e00000 } |.endif } | jmp >4 |.code |4: } if (needs_slow_path) { |.cold_code |7: | SET_EX_OPLINE opline, r0 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2a, name |.if X64 | mov CARG3, EX->run_time_cache | add CARG3, opline->extended_value if (opline->result_type == IS_UNUSED) { | xor CARG4, CARG4 } else { | LOAD_ZVAL_ADDR CARG4, res_addr } |.else | sub r4, 8 if (opline->result_type == IS_UNUSED) { | push 0 } else { | PUSH_ZVAL_ADDR res_addr, r0 } | mov r0, EX->run_time_cache | add r0, opline->extended_value | push r0 |.endif switch (opline->opcode) { case ZEND_PRE_INC_OBJ: | EXT_CALL zend_jit_pre_inc_obj_helper, r0 break; case ZEND_PRE_DEC_OBJ: | EXT_CALL zend_jit_pre_dec_obj_helper, r0 break; case ZEND_POST_INC_OBJ: | EXT_CALL zend_jit_post_inc_obj_helper, r0 break; case ZEND_POST_DEC_OBJ: | EXT_CALL zend_jit_post_dec_obj_helper, r0 break; default: ZEND_UNREACHABLE(); } |.if not(X64) | add r4, 8 |.endif | jmp >9 |.code } |9: if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_assign_obj_op(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t val_info, zend_ssa_range *val_range, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool use_this, zend_class_entry *trace_ce, int may_throw) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; bool needs_slow_path = 0; binary_op_type binary_op = get_binary_op(opline->extended_value); ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); ZEND_ASSERT(opline->result_type == IS_UNUSED); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { | GET_ZVAL_PTR FCARG1a, this_addr } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 | GET_Z_PTR FCARG1a, FCARG1a |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_REF) { if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | ZVAL_DEREF FCARG1a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 |.cold_code |1: | SET_EX_OPLINE opline, r0 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | LOAD_ADDR FCARG2a, ZSTR_VAL(name) if (op1_info & MAY_BE_UNDEF) { | EXT_CALL zend_jit_invalid_property_assign_op, r0 } else { | EXT_CALL zend_jit_invalid_property_assign, r0 } if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | jmp >8 } else { | jmp ->exception_handler } |.code } } | GET_ZVAL_PTR FCARG1a, op1_addr } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (!zend_jit_class_guard(Dst, opline, trace_ce)) { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } if (!prop_info) { needs_slow_path = 1; | mov r0, EX->run_time_cache | mov r2, aword [r0 + (opline+1)->extended_value] | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >7 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | cmp aword [r0 + ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2], 0 | jnz >7 } | mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)] | test r0, r0 | jl >7 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 | add FCARG1a, r0 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr } else { | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 needs_slow_path = 1; } if (ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t info = val_info; if (opline) { | SET_EX_OPLINE opline, r0 } | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1 |.cold_code |1: | GET_ZVAL_PTR FCARG1a, prop_addr if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2a, val_addr } |.if X64 | LOAD_ADDR CARG3, binary_op |.else | sub r4, 12 | PUSH_ADDR binary_op, r0 |.endif | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 |.if not(X64) | add r4, 12 |.endif | jmp >9 |.code | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2a, prop_info } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] | mov FCARG2a, aword[r0 + prop_info_offset] } | LOAD_ZVAL_ADDR FCARG1a, prop_addr |.if X64 | LOAD_ZVAL_ADDR CARG3, val_addr | LOAD_ADDR CARG4, binary_op |.else | sub r4, 8 | PUSH_ADDR binary_op, r0 | PUSH_ZVAL_ADDR val_addr, r0 |.endif | EXT_CALL zend_jit_assign_op_to_typed_prop, r0 |.if not(X64) | add r4, 8 |.endif if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { info |= MAY_BE_RC1|MAY_BE_RCN; } | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { zend_jit_addr var_addr = prop_addr; uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); | LOAD_ZVAL_ADDR r0, prop_addr | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 | GET_ZVAL_PTR FCARG1a, var_addr | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 | jnz >1 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] |.cold_code |1: if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2a, val_addr } if (opline) { | SET_EX_OPLINE opline, r0 } |.if X64 | LOAD_ADDR CARG3, binary_op |.else | sub r4, 12 | PUSH_ADDR binary_op, r0 |.endif | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 |.if not(X64) | add r4, 12 |.endif | jmp >9 |.code |2: switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_DIV: if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 1 /* may overflow */, 0)) { return 0; } break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, NULL, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, val_range, 0, var_addr, var_def_info, var_info, 0)) { return 0; } break; case ZEND_CONCAT: if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 0)) { return 0; } break; default: ZEND_UNREACHABLE(); } } if (needs_slow_path) { |.cold_code |7: | SET_EX_OPLINE opline, r0 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2a, name |.if X64 | LOAD_ZVAL_ADDR CARG3, val_addr | mov CARG4, EX->run_time_cache | add CARG4, (opline+1)->extended_value |.if X64WIN | LOAD_ADDR r0, binary_op | mov aword A5, r0 |.else | LOAD_ADDR CARG5, binary_op |.endif |.else | sub r4, 4 | PUSH_ADDR binary_op, r0 | mov r0, EX->run_time_cache | add r0, (opline+1)->extended_value | push r0 | PUSH_ZVAL_ADDR val_addr, r0 |.endif | EXT_CALL zend_jit_assign_obj_op_helper, r0 |.if not(X64) | add r4, 4 |.endif if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1|MAY_BE_RCN; } |8: | // FREE_OP_DATA(); | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline | jmp >9 |.code } |9: if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_assign_obj(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t val_info, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool use_this, zend_class_entry *trace_ce, int may_throw) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr res_addr = 0; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; bool needs_slow_path = 0; if (RETURN_VALUE_USED(opline)) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { | GET_ZVAL_PTR FCARG1a, this_addr } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 | GET_Z_PTR FCARG1a, FCARG1a |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & MAY_BE_REF) { if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | ZVAL_DEREF FCARG1a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 |.cold_code |1: | SET_EX_OPLINE opline, r0 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr } | LOAD_ADDR FCARG2a, ZSTR_VAL(name) | EXT_CALL zend_jit_invalid_property_assign, r0 if (RETURN_VALUE_USED(opline)) { | SET_ZVAL_TYPE_INFO res_addr, IS_NULL } if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | jmp >8 } else { | jmp ->exception_handler } |.code } } | GET_ZVAL_PTR FCARG1a, op1_addr } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (!zend_jit_class_guard(Dst, opline, trace_ce)) { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } if (!prop_info) { needs_slow_path = 1; | mov r0, EX->run_time_cache | mov r2, aword [r0 + opline->extended_value] | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] | jne >5 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] } | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] | test r0, r0 | jl >5 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5 | add FCARG1a, r0 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | test FCARG2a, FCARG2a | jnz >1 |.cold_code |1: | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); | SET_EX_OPLINE opline, r0 |.if X64 | LOAD_ZVAL_ADDR CARG3, val_addr if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR CARG4, res_addr } else { | xor CARG4, CARG4 } |.else | sub r4, 8 if (RETURN_VALUE_USED(opline)) { | PUSH_ZVAL_ADDR res_addr, r0 } else { | push 0 } | PUSH_ZVAL_ADDR val_addr, r0 |.endif | EXT_CALL zend_jit_assign_to_typed_prop, r0 |.if not(X64) | add r4, 8 |.endif if ((opline+1)->op1_type == IS_CONST) { | // TODO: ??? | // if (Z_TYPE_P(value) == orig_type) { | // CACHE_PTR_EX(cache_slot + 2, NULL); } if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | jmp >8 } else { | jmp >9 } |.code } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set) { // Undefined property with magic __get()/__set() if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr } else { | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5 needs_slow_path = 1; } } if (ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t info = val_info; | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); | SET_EX_OPLINE opline, r0 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2a, prop_info } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] | mov FCARG2a, aword[r0 + prop_info_offset] } | LOAD_ZVAL_ADDR FCARG1a, prop_addr |.if X64 | LOAD_ZVAL_ADDR CARG3, val_addr if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR CARG4, res_addr } else { | xor CARG4, CARG4 } |.else | sub r4, 8 if (RETURN_VALUE_USED(opline)) { | PUSH_ZVAL_ADDR res_addr, r0 } else { | push 0 } | PUSH_ZVAL_ADDR val_addr, r0 |.endif | EXT_CALL zend_jit_assign_to_typed_prop, r0 |.if not(X64) | add r4, 8 |.endif if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { info |= MAY_BE_RC1|MAY_BE_RCN; } | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); if (opline->result_type == IS_UNUSED) { if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { return 0; } } else { if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { return 0; } } } if (needs_slow_path) { |.cold_code |5: | SET_EX_OPLINE opline, r0 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2a, name |.if X64 | LOAD_ZVAL_ADDR CARG3, val_addr | mov CARG4, EX->run_time_cache | add CARG4, opline->extended_value if (RETURN_VALUE_USED(opline)) { |.if X64WIN | LOAD_ZVAL_ADDR r0, res_addr | mov aword A5, r0 |.else | LOAD_ZVAL_ADDR CARG5, res_addr |.endif } else { |.if X64WIN | mov aword A5, 0 |.else | xor CARG5, CARG5 |.endif } |.else | sub r4, 4 if (RETURN_VALUE_USED(opline)) { | PUSH_ZVAL_ADDR res_addr, r0 } else { | push 0 } | mov r0, EX->run_time_cache | add r0, opline->extended_value | push r0 | PUSH_ZVAL_ADDR val_addr, r0 |.endif | EXT_CALL zend_jit_assign_obj_helper, r0 |.if not(X64) | add r4, 4 |.endif if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1|MAY_BE_RCN; } |8: | // FREE_OP_DATA(); | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline | jmp >9 |.code } |9: if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) { zend_jit_addr op1_addr = OP1_ADDR(); if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (may_throw) { | SET_EX_OPLINE opline, r0 } if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { if (op1_info & MAY_BE_ARRAY) { | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 } | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] | cmp FCARG1d, -1 | je >7 | EXT_CALL zend_hash_iterator_del, r0 |7: } | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } } return 1; } static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { if (opline->op1_type == IS_CONST) { zval *zv; size_t len; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); len = Z_STRLEN_P(zv); if (len > 0) { const char *str = Z_STRVAL_P(zv); | SET_EX_OPLINE opline, r0 |.if X64 | LOAD_ADDR CARG1, str | LOAD_ADDR CARG2, len | EXT_CALL zend_write, r0 |.else | mov aword A2, len | mov aword A1, str | EXT_CALL zend_write, r0 |.endif if (!zend_jit_check_exception(Dst)) { return 0; } } } else { zend_jit_addr op1_addr = OP1_ADDR(); ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); | SET_EX_OPLINE opline, r0 | GET_ZVAL_PTR r0, op1_addr |.if X64 | lea CARG1, aword [r0 + offsetof(zend_string, val)] | mov CARG2, aword [r0 + offsetof(zend_string, len)] | EXT_CALL zend_write, r0 |.else | add r0, offsetof(zend_string, val) | mov aword A1, r0 | mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))] | mov aword A2, r0 | EXT_CALL zend_write, r0 |.endif if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline } if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) { zend_jit_addr res_addr = RES_ADDR(); if (opline->op1_type == IS_CONST) { zval *zv; size_t len; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); len = Z_STRLEN_P(zv); | SET_ZVAL_LVAL res_addr, len | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } else { ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); | GET_ZVAL_PTR r0, op1_addr | mov r0, aword [r0 + offsetof(zend_string, len)] | SET_ZVAL_LVAL res_addr, r0 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline } return 1; } static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, int may_throw) { zend_jit_addr res_addr = RES_ADDR(); if (opline->op1_type == IS_CONST) { zval *zv; zend_long count; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); count = zend_hash_num_elements(Z_ARRVAL_P(zv)); | SET_ZVAL_LVAL res_addr, count | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } else { ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. | GET_ZVAL_PTR r0, op1_addr // Sign-extend the 32-bit value to a potentially 64-bit zend_long | mov eax, dword [r0 + offsetof(HashTable, nNumOfElements)] | SET_ZVAL_LVAL res_addr, r0 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline } if (may_throw) { return zend_jit_check_exception(Dst); } return 1; } static int zend_jit_load_this(dasm_State **Dst, uint32_t var) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); | mov FCARG1a, aword EX->This.value.ptr | SET_ZVAL_PTR var_addr, FCARG1a | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX | GC_ADDREF FCARG1a return 1; } static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) { if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (!JIT_G(current_frame) || !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); | cmp byte EX->This.u1.v.type, IS_OBJECT | jne &exit_addr if (JIT_G(current_frame)) { TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); } } } else { | cmp byte EX->This.u1.v.type, IS_OBJECT | jne >1 |.cold_code |1: | SET_EX_OPLINE opline, r0 | jmp ->invalid_this |.code } } if (!check_only) { if (!zend_jit_load_this(Dst, opline->result.var)) { return 0; } } return 1; } static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) { uint32_t count; Bucket *p; const zend_op *target; int b; int32_t exit_point; const void *exit_addr; | test r0, r0 if (default_label) { | jz &default_label } else if (next_opline) { | jz >3 } else { | jz =>default_b } | LOAD_ADDR FCARG1a, jumptable | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) |.if X64 | cqo |.else | cdq |.endif | idiv FCARG1a |.if X64 if (!IS_32BIT(dasm_end)) { | lea FCARG1a, aword [>4] | jmp aword [FCARG1a + r0] } else { | jmp aword [r0 + >4] } |.else | jmp aword [r0 + >4] |.endif |.jmp_table |.align aword |4: if (trace_info) { trace_info->jmp_table_size += zend_hash_num_elements(jumptable); } count = jumptable->nNumUsed; p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { if (default_label) { | .aword &default_label } else if (next_opline) { | .aword >3 } else { | .aword =>default_b } } else { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); if (!next_opline) { b = ssa->cfg.map[target - op_array->opcodes]; | .aword =>b } else if (next_opline == target) { | .aword >3 } else { exit_point = zend_jit_trace_get_exit_point(target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); | .aword &exit_addr } } p++; count--; } while (count); |.code return 1; } static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) { HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); const zend_op *next_opline = NULL; if (trace) { ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); ZEND_ASSERT(trace->opline != NULL); next_opline = trace->opline; } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); zval *jump_zv = NULL; int b; if (opline->opcode == ZEND_SWITCH_LONG) { if (Z_TYPE_P(zv) == IS_LONG) { jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); } } else if (opline->opcode == ZEND_SWITCH_STRING) { if (Z_TYPE_P(zv) == IS_STRING) { jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); } } else if (opline->opcode == ZEND_MATCH) { if (Z_TYPE_P(zv) == IS_LONG) { jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); } else if (Z_TYPE_P(zv) == IS_STRING) { jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); } } else { ZEND_UNREACHABLE(); } if (next_opline) { const zend_op *target; if (jump_zv != NULL) { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); } else { target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); } ZEND_ASSERT(target == next_opline); } else { if (jump_zv != NULL) { b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; } else { b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; } | jmp =>b } } else { zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr = OP1_ADDR(); const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); const zend_op *target; int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; int b; int32_t exit_point; const void *fallback_label = NULL; const void *default_label = NULL; const void *exit_addr; if (next_opline) { if (next_opline != opline + 1) { exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); } if (next_opline != default_opline) { exit_point = zend_jit_trace_get_exit_point(default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); } } if (opline->opcode == ZEND_SWITCH_LONG) { if (op1_info & MAY_BE_LONG) { if (op1_info & MAY_BE_REF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1 | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr |.cold_code |1: | // ZVAL_DEREF(op) if (fallback_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 } | GET_ZVAL_PTR FCARG2a, op1_addr if (fallback_label) { | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label } else { | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 } | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] | jmp >2 |.code |2: } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (fallback_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 } } | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr } if (HT_IS_PACKED(jumptable)) { uint32_t count = jumptable->nNumUsed; Bucket *p = jumptable->arData; | cmp FCARG2a, jumptable->nNumUsed if (default_label) { | jae &default_label } else if (next_opline) { | jae >3 } else { | jae =>default_b } |.if X64 if (!IS_32BIT(dasm_end)) { | lea r0, aword [>4] | jmp aword [r0 + FCARG2a * 8] } else { | jmp aword [FCARG2a * 8 + >4] } |.else | jmp aword [FCARG2a * 4 + >4] |.endif |.jmp_table |.align aword |4: if (trace_info) { trace_info->jmp_table_size += count; } p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { if (default_label) { | .aword &default_label } else if (next_opline) { | .aword >3 } else { | .aword =>default_b } } else { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); if (!next_opline) { b = ssa->cfg.map[target - op_array->opcodes]; | .aword =>b } else if (next_opline == target) { | .aword >3 } else { exit_point = zend_jit_trace_get_exit_point(target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); | .aword &exit_addr } } p++; count--; } while (count); |.code |3: } else { | LOAD_ADDR FCARG1a, jumptable | EXT_CALL zend_hash_index_find, r0 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { return 0; } |3: } } } else if (opline->opcode == ZEND_SWITCH_STRING) { if (op1_info & MAY_BE_STRING) { if (op1_info & MAY_BE_REF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1 | GET_ZVAL_PTR FCARG2a, op1_addr |.cold_code |1: | // ZVAL_DEREF(op) if (fallback_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 } | GET_ZVAL_PTR FCARG2a, op1_addr if (fallback_label) { | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label } else { | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 } | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] | jmp >2 |.code |2: } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { if (fallback_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 } } | GET_ZVAL_PTR FCARG2a, op1_addr } | LOAD_ADDR FCARG1a, jumptable | EXT_CALL zend_hash_find, r0 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { return 0; } |3: } } else if (opline->opcode == ZEND_MATCH) { if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG2a, op1_addr | ZVAL_DEREF FCARG2a, op1_info op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); } | LOAD_ADDR FCARG1a, jumptable if (op1_info & MAY_BE_LONG) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (op1_info & MAY_BE_STRING) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5 } else if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 } else if (default_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b } } | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr | EXT_CALL zend_hash_index_find, r0 if (op1_info & MAY_BE_STRING) { | jmp >2 } } if (op1_info & MAY_BE_STRING) { |5: if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 } else if (default_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b } } | GET_ZVAL_PTR FCARG2a, op1_addr | EXT_CALL zend_hash_find, r0 } |2: if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { return 0; } } if (op1_info & MAY_BE_UNDEF) { |6: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { if (default_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b } } | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, r0 | mov FCARG1d, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, r0 if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } if (default_label) { | jmp &default_label } else if (next_opline) { | jmp >3 } else { | jmp =>default_b } |3: } else { ZEND_UNREACHABLE(); } } return 1; } static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) { zend_arg_info *arg_info = &op_array->arg_info[-1]; ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); zend_jit_addr op1_addr = OP1_ADDR(); bool needs_slow_check = 1; bool slow_check_in_cold = 1; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; if (type_mask == 0) { slow_check_in_cold = 0; } else { if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { slow_check_in_cold = 0; } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { needs_slow_check = 0; } else if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); | IF_NOT_ZVAL_TYPE op1_addr, type_code, >7 } else { | mov edx, 1 | GET_ZVAL_TYPE cl, op1_addr | shl edx, cl | test edx, type_mask | je >7 } } if (needs_slow_check) { if (slow_check_in_cold) { |.cold_code |7: } | SET_EX_OPLINE opline, r1 if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >8 | mov FCARG1a, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, FCARG2a | LOAD_ADDR_ZTS r0, executor_globals, uninitialized_zval } |8: | LOAD_ZVAL_ADDR FCARG1a, op1_addr | mov FCARG2a, EX->func |.if X64 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info | mov r0, EX->run_time_cache | lea CARG4, aword [r0+opline->op2.num] | EXT_CALL zend_jit_verify_return_slow, r0 |.else | sub r4, 8 | mov r0, EX->run_time_cache | add r0, opline->op2.num | push r0 | push (ptrdiff_t)arg_info | EXT_CALL zend_jit_verify_return_slow, r0 | add r4, 8 |.endif if (!zend_jit_check_exception(Dst)) { return 0; } if (slow_check_in_cold) { | jmp >9 |.code } } |9: return 1; } static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); if (op1_info & MAY_BE_REF) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, op1_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); } | ZVAL_DEREF FCARG1a, op1_info |1: } if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { if (exit_addr) { ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPNZ) { | jmp =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | jmp =>target_label2 } } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE } } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { if (exit_addr) { ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); } else if (smart_branch_opcode) { if (smart_branch_opcode != ZEND_JMPNZ) { | jmp =>target_label } } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE } } else { ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); | cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | jg &exit_addr } else { | jle &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | jle =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | jg =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | jle =>target_label | jmp =>target_label2 } else { ZEND_UNREACHABLE(); } } else { | setg al | movzx eax, al | lea eax, [eax + IS_FALSE] | SET_ZVAL_TYPE_INFO res_addr, eax } } return 1; } static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, r0 } } else { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); | // ZVAL_COPY(res, value); | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1a if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, ah, FCARG1a } } | // Z_FE_POS_P(res) = 0; | mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0 return 1; } static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); | // array = EX_VAR(opline->op1.var); | // fe_ht = Z_ARRVAL_P(array); | GET_ZVAL_PTR FCARG1a, op1_addr | // pos = Z_FE_POS_P(array); | mov eax, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)] | // p = fe_ht->arData + pos; |.if X64 || ZEND_ASSERT(sizeof(Bucket) == 32); | mov FCARG2d, eax | shl FCARG2a, 5 |.else | imul FCARG2a, r0, sizeof(Bucket) |.endif | add FCARG2a, aword [FCARG1a + offsetof(zend_array, arData)] |1: | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); | // ZEND_VM_CONTINUE(); if (exit_addr) { if (exit_opcode == ZEND_JMP) { | jbe &exit_addr } else { | jbe >3 } } else { | jbe =>target_label } | // pos++; | add eax, 1 | // value_type = Z_TYPE_INFO_P(value); | // if (EXPECTED(value_type != IS_UNDEF)) { if (!exit_addr || exit_opcode == ZEND_JMP) { | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >3 } else { | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr } | // p++; | add FCARG2a, sizeof(Bucket) | jmp <1 |3: if (!exit_addr || exit_opcode == ZEND_JMP) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); uint32_t val_info; | // Z_FE_POS_P(array) = pos + 1; | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax if (RETURN_VALUE_USED(opline)) { zend_jit_addr res_addr = RES_ADDR(); if ((op1_info & MAY_BE_ARRAY_KEY_LONG) && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { | // if (!p->key) { | cmp aword [FCARG2a + offsetof(Bucket, key)], 0 | jz >2 } if (op1_info & MAY_BE_ARRAY_KEY_STRING) { | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); | mov r0, aword [FCARG2a + offsetof(Bucket, key)] | SET_ZVAL_PTR res_addr, r0 | test dword [r0 + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED | jz >1 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING | jmp >3 |1: | GC_ADDREF r0 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | jmp >3 |2: } } if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); | mov r0, aword [FCARG2a + offsetof(Bucket, h)] | SET_ZVAL_LVAL res_addr, r0 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG } |3: } val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); if (val_info & MAY_BE_ARRAY) { val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } if (op1_info & MAY_BE_ARRAY_OF_REF) { val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1 | MAY_BE_RCN; } if (opline->op2_type == IS_CV) { | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { return 0; } } else { | // ZVAL_COPY(res, value); | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1a | TRY_ADDREF val_info, ah, FCARG1a } } return 1; } static int zend_jit_fetch_constant(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op) { zval *zv = RT_CONSTANT(opline, opline->op2) + 1; zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); uint32_t res_info = RES_INFO(); | // c = CACHED_PTR(opline->extended_value); | mov FCARG1a, EX->run_time_cache | mov r0, aword [FCARG1a + opline->extended_value] | // if (c != NULL) | test r0, r0 | jz >9 | // if (!IS_SPECIAL_CACHE_VAL(c)) | test r0, CACHE_SPECIAL | jnz >9 |8: if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); int32_t exit_point; const void *exit_addr = NULL; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); exit_point = zend_jit_trace_get_exit_point(opline+1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; zend_uchar type = concrete_type(res_info); if (type < IS_STRING) { | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr } else { | GET_ZVAL_TYPE_INFO edx, const_addr | IF_NOT_TYPE dl, type, &exit_addr } | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1 if (type < IS_STRING) { | SET_ZVAL_TYPE_INFO res_addr, type } else { | SET_ZVAL_TYPE_INFO res_addr, edx | TRY_ADDREF res_info, dh, r1 } } else { | // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1 | TRY_ADDREF MAY_BE_ANY, ah, r1 } |.cold_code |9: | // SAVE_OPLINE(); | SET_EX_OPLINE opline, r0 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); | LOAD_ADDR FCARG1a, zv | mov FCARG2a, opline->op1.num | EXT_CALL zend_jit_get_constant, r0 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); | test r0, r0 | jnz <8 | jmp ->exception_handler |.code return 1; } static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); | LOAD_ADDR FCARG1a, ht if (opline->op1_type != IS_CONST) { | GET_ZVAL_PTR FCARG2a, op1_addr | EXT_CALL zend_hash_find, r0 } else { zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); | LOAD_ADDR FCARG2a, str | EXT_CALL _zend_hash_find_known_hash, r0 } | test r0, r0 if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | jz &exit_addr } else { | jnz &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | jz =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | jnz =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | jz =>target_label | jmp =>target_label2 } else { ZEND_UNREACHABLE(); } } else { | setnz al | movzx eax, al | lea eax, [eax + IS_FALSE] | SET_ZVAL_TYPE_INFO res_addr, eax } return 1; } static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr return 1; } static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; const void *exit_addr = NULL; if (add_ref_guard || add_type_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if (add_ref_guard) { | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr } if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ if (Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1a, var_addr } | EXT_CALL zend_jit_unref_helper, r0 } else { | GET_ZVAL_PTR FCARG1a, var_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val)); *var_addr_ptr = var_addr; } if (var_type != IS_UNKNOWN) { var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); } if (add_type_guard && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { var_info = (1 << var_type); } else if (var_type != IS_ARRAY) { var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); } else { var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); } *var_info_ptr = var_info; } else { var_info &= ~MAY_BE_REF; *var_info_ptr = var_info; } return 1; } static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; int32_t exit_point; const void *exit_addr; if (add_indirect_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr | GET_ZVAL_PTR FCARG1a, var_addr } else { /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ if (opline->op1_type != IS_VAR || (opline-1)->result_type != IS_VAR || (opline-1)->result.var != opline->op1.var || (opline-1)->op2_type == IS_VAR || (opline-1)->op2_type == IS_TMP_VAR) { | GET_ZVAL_PTR FCARG1a, var_addr } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { | mov FCARG1a, r0 } } *var_info_ptr &= ~MAY_BE_INDIRECT; var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); *var_addr_ptr = var_addr; if (var_type != IS_UNKNOWN) { var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); } if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { var_info = (1 << var_type); } else if (var_type != IS_ARRAY) { var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); } else { var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); } *var_info_ptr = var_info; } return 1; } static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) { if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { return 0; } switch (opline->opcode) { case ZEND_QM_ASSIGN: case ZEND_SEND_VAR: case ZEND_ASSIGN: case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: return 1; case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: if (def_var == ssa_op->result_def && use_var == ssa_op->op1_use) { return 1; } break; default: break; } return 0; } static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) { uint32_t op1_info, op2_info; switch (opline->opcode) { case ZEND_QM_ASSIGN: case ZEND_SEND_VAR: case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: case ZEND_CASE: case ZEND_RETURN: return 1; case ZEND_ASSIGN: op1_info = OP1_INFO(); op2_info = OP2_INFO(); return opline->op1_type == IS_CV && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: op1_info = OP1_INFO(); op2_info = OP2_INFO(); return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: op1_info = OP1_INFO(); op2_info = OP2_INFO(); return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: op1_info = OP1_INFO(); return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: return 1; case ZEND_FETCH_DIM_R: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (trace && trace->op1_type != IS_UNKNOWN && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); } return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); } return 0; } static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) { if (ssa->vars[var].no_val) { /* we don't need the value */ return 0; } if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { /* Disable global register allocation, * register allocation for SSA variables connected through Phi functions */ if (ssa->vars[var].definition_phi) { return 0; } if (ssa->vars[var].phi_use_chain) { zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; do { if (!ssa->vars[phi->ssa_var].no_val) { return 0; } phi = zend_ssa_next_use_phi(ssa, var, phi); } while (phi); } } if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { /* bad type */ return 0; } return 1; } static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) { if (!zend_jit_var_supports_reg(ssa, var)) { return 0; } if (ssa->vars[var].definition >= 0) { uint32_t def = ssa->vars[var].definition; if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { return 0; } } if (ssa->vars[var].use_chain >= 0) { int use = ssa->vars[var].use_chain; do { if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { return 0; } use = zend_ssa_next_use(ssa->ops, var, use); } while (use >= 0); } return 1; } static bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op) { |.if X64 || if (op_type == IS_CONST) { || zval *zv = RT_CONSTANT(opline, op); || if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) { || return 1; || } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) { || return 1; || } || } |.endif return 0; } static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) { uint32_t op1_info, op2_info; switch (opline->opcode) { case ZEND_FETCH_DIM_R: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { return ZEND_REGSET(ZREG_FCARG1a); } break; default: break; } return ZEND_REGSET_EMPTY; } static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) { uint32_t op1_info, op2_info, res_info; zend_regset regset = ZEND_REGSET_SCRATCH; switch (opline->opcode) { case ZEND_NOP: case ZEND_OP_DATA: case ZEND_JMP: case ZEND_RETURN: regset = ZEND_REGSET_EMPTY; break; case ZEND_QM_ASSIGN: if (ssa_op->op1_def == current_var || ssa_op->result_def == current_var) { regset = ZEND_REGSET_EMPTY; break; } /* break missing intentionally */ case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: if (ssa_op->op1_use == current_var) { regset = ZEND_REGSET(ZREG_R0); break; } op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_R0); } else { regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); } } break; case ZEND_SEND_VAR: if (ssa_op->op1_use == current_var || ssa_op->op1_def == current_var) { regset = ZEND_REGSET_EMPTY; break; } op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { } else { regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); if (op1_info & MAY_BE_REF) { ZEND_REGSET_INCL(regset, ZREG_R1); } } } break; case ZEND_ASSIGN: if (ssa_op->op2_use == current_var || ssa_op->op2_def == current_var || ssa_op->op1_def == current_var || ssa_op->result_def == current_var) { regset = ZEND_REGSET_EMPTY; break; } op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (opline->op1_type == IS_CV && !(op2_info & MAY_BE_UNDEF) && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_R0); } else { regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); } } break; case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: if (ssa_op->op1_use == current_var || ssa_op->op1_def == current_var || ssa_op->result_def == current_var) { regset = ZEND_REGSET_EMPTY; break; } op1_info = OP1_INFO(); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_LONG) && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; if (op1_info & MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_XMM0); } } break; case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_R0); } res_info = OP1_INFO(); if (res_info & MAY_BE_DOUBLE) { ZEND_REGSET_INCL(regset, ZREG_XMM0); ZEND_REGSET_INCL(regset, ZREG_XMM1); } } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { if (ssa_op->result_def != current_var) { ZEND_REGSET_INCL(regset, ZREG_XMM0); } } if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { if (zend_is_commutative(opline->opcode)) { if (ssa_op->result_def != current_var) { ZEND_REGSET_INCL(regset, ZREG_XMM0); } } else { ZEND_REGSET_INCL(regset, ZREG_XMM0); if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_XMM1); } } } if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use) && (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_XMM0); } } if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { if (!ZEND_REGSET_IN(regset, ZREG_R0)) { ZEND_REGSET_INCL(regset, ZREG_R0); } else { ZEND_REGSET_INCL(regset, ZREG_R1); } } } break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_R0); } if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { if (!ZEND_REGSET_IN(regset, ZREG_R0)) { ZEND_REGSET_INCL(regset, ZREG_R0); } else { ZEND_REGSET_INCL(regset, ZREG_R1); } } } break; case ZEND_SL: case ZEND_SR: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_R0); } if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { ZEND_REGSET_INCL(regset, ZREG_R1); } } break; case ZEND_MOD: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; if (opline->op2_type == IS_CONST && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && OP1_HAS_RANGE() && OP1_MIN_RANGE() >= 0) { if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_R0); } if (sizeof(void*) == 8 && !IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) - 1)) { if (!ZEND_REGSET_IN(regset, ZREG_R0)) { ZEND_REGSET_INCL(regset, ZREG_R0); } else { ZEND_REGSET_INCL(regset, ZREG_R1); } } } else { ZEND_REGSET_INCL(regset, ZREG_R0); ZEND_REGSET_INCL(regset, ZREG_R2); if (opline->op2_type == IS_CONST) { ZEND_REGSET_INCL(regset, ZREG_R1); } } } break; case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: case ZEND_CASE: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { ZEND_REGSET_INCL(regset, ZREG_R0); } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { if (ssa_op->op1_use != current_var && ssa_op->op2_use != current_var) { ZEND_REGSET_INCL(regset, ZREG_R0); } } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { ZEND_REGSET_INCL(regset, ZREG_XMM0); } if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { ZEND_REGSET_INCL(regset, ZREG_XMM0); } if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { if (ssa_op->op1_use != current_var && ssa_op->op2_use != current_var) { ZEND_REGSET_INCL(regset, ZREG_XMM0); } } if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { ZEND_REGSET_INCL(regset, ZREG_R0); } } break; case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: op1_info = OP1_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; if (op1_info & MAY_BE_DOUBLE) { ZEND_REGSET_INCL(regset, ZREG_XMM0); } if (opline->opcode == ZEND_BOOL || opline->opcode == ZEND_BOOL_NOT || opline->opcode == ZEND_JMPZ_EX || opline->opcode == ZEND_JMPNZ_EX) { ZEND_REGSET_INCL(regset, ZREG_R0); } } break; case ZEND_DO_UCALL: case ZEND_DO_FCALL: case ZEND_DO_FCALL_BY_NAME: case ZEND_INCLUDE_OR_EVAL: case ZEND_GENERATOR_CREATE: case ZEND_YIELD: case ZEND_YIELD_FROM: regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); break; default: break; } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (ssa_op == ssa->ops && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { ZEND_REGSET_INCL(regset, ZREG_R0); ZEND_REGSET_INCL(regset, ZREG_R1); } } /* %r0 is used to check EG(vm_interrupt) */ if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (ssa_op == ssa->ops && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { #if ZTS ZEND_REGSET_INCL(regset, ZREG_R0); #else if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { ZEND_REGSET_INCL(regset, ZREG_R0); } #endif } } else { uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { #if ZTS ZEND_REGSET_INCL(regset, ZREG_R0); #else if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { ZEND_REGSET_INCL(regset, ZREG_R0); } #endif } } return regset; } #if defined(__clang__) # pragma clang diagnostic pop #endif /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * indent-tabs-mode: t * End: */