diff options
Diffstat (limited to 'deps/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h')
| -rw-r--r-- | deps/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h | 246 |
1 files changed, 50 insertions, 196 deletions
diff --git a/deps/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h b/deps/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h index d5cda7b3c4..50032eac23 100644 --- a/deps/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h +++ b/deps/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h @@ -1317,7 +1317,9 @@ void LiftoffAssembler::emit_i64_addi(LiftoffRegister dst, LiftoffRegister lhs, void LiftoffAssembler::emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { - if (dst.gp() == rhs.gp()) { + if (lhs.gp() == rhs.gp()) { + xorq(dst.gp(), dst.gp()); + } else if (dst.gp() == rhs.gp()) { negq(dst.gp()); addq(dst.gp(), lhs.gp()); } else { @@ -2335,29 +2337,6 @@ void EmitSimdShiftOpImm(LiftoffAssembler* assm, LiftoffRegister dst, } } -template <bool is_signed> -void EmitI8x16Shr(LiftoffAssembler* assm, LiftoffRegister dst, - LiftoffRegister lhs, LiftoffRegister rhs) { - // Same algorithm as the one in code-generator-x64.cc. - assm->Punpckhbw(kScratchDoubleReg, lhs.fp()); - assm->Punpcklbw(dst.fp(), lhs.fp()); - // Prepare shift value - assm->movq(kScratchRegister, rhs.gp()); - // Take shift value modulo 8. - assm->andq(kScratchRegister, Immediate(7)); - assm->addq(kScratchRegister, Immediate(8)); - assm->Movq(liftoff::kScratchDoubleReg2, kScratchRegister); - if (is_signed) { - assm->Psraw(kScratchDoubleReg, liftoff::kScratchDoubleReg2); - assm->Psraw(dst.fp(), liftoff::kScratchDoubleReg2); - assm->Packsswb(dst.fp(), kScratchDoubleReg); - } else { - assm->Psrlw(kScratchDoubleReg, liftoff::kScratchDoubleReg2); - assm->Psrlw(dst.fp(), liftoff::kScratchDoubleReg2); - assm->Packuswb(dst.fp(), kScratchDoubleReg); - } -} - inline void EmitAnyTrue(LiftoffAssembler* assm, LiftoffRegister dst, LiftoffRegister src) { assm->xorq(dst.gp(), dst.gp()); @@ -2414,21 +2393,11 @@ void LiftoffAssembler::LoadTransform(LiftoffRegister dst, Register src_addr, } else { DCHECK_EQ(LoadTransformationKind::kSplat, transform); if (memtype == MachineType::Int8()) { - Pinsrb(dst.fp(), dst.fp(), src_op, 0); - Pxor(kScratchDoubleReg, kScratchDoubleReg); - Pshufb(dst.fp(), kScratchDoubleReg); + S128Load8Splat(dst.fp(), src_op, kScratchDoubleReg); } else if (memtype == MachineType::Int16()) { - Pinsrw(dst.fp(), dst.fp(), src_op, 0); - Pshuflw(dst.fp(), dst.fp(), uint8_t{0}); - Punpcklqdq(dst.fp(), dst.fp()); + S128Load16Splat(dst.fp(), src_op, kScratchDoubleReg); } else if (memtype == MachineType::Int32()) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vbroadcastss(dst.fp(), src_op); - } else { - movss(dst.fp(), src_op); - shufps(dst.fp(), dst.fp(), byte{0}); - } + S128Load32Splat(dst.fp(), src_op); } else if (memtype == MachineType::Int64()) { Movddup(dst.fp(), src_op); } @@ -2440,18 +2409,17 @@ void LiftoffAssembler::LoadLane(LiftoffRegister dst, LiftoffRegister src, uintptr_t offset_imm, LoadType type, uint8_t laneidx, uint32_t* protected_load_pc) { Operand src_op = liftoff::GetMemOp(this, addr, offset_reg, offset_imm); - *protected_load_pc = pc_offset(); MachineType mem_type = type.mem_type(); if (mem_type == MachineType::Int8()) { - Pinsrb(dst.fp(), src.fp(), src_op, laneidx); + Pinsrb(dst.fp(), src.fp(), src_op, laneidx, protected_load_pc); } else if (mem_type == MachineType::Int16()) { - Pinsrw(dst.fp(), src.fp(), src_op, laneidx); + Pinsrw(dst.fp(), src.fp(), src_op, laneidx, protected_load_pc); } else if (mem_type == MachineType::Int32()) { - Pinsrd(dst.fp(), src.fp(), src_op, laneidx); + Pinsrd(dst.fp(), src.fp(), src_op, laneidx, protected_load_pc); } else { DCHECK_EQ(MachineType::Int64(), mem_type); - Pinsrq(dst.fp(), src.fp(), src_op, laneidx); + Pinsrq(dst.fp(), src.fp(), src_op, laneidx, protected_load_pc); } } @@ -2515,26 +2483,24 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, void LiftoffAssembler::emit_i8x16_swizzle(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { - I8x16Swizzle(dst.fp(), lhs.fp(), rhs.fp()); + I8x16Swizzle(dst.fp(), lhs.fp(), rhs.fp(), kScratchDoubleReg, + kScratchRegister); } void LiftoffAssembler::emit_i8x16_popcnt(LiftoffRegister dst, LiftoffRegister src) { - I8x16Popcnt(dst.fp(), src.fp(), liftoff::kScratchDoubleReg2); + I8x16Popcnt(dst.fp(), src.fp(), kScratchDoubleReg, + liftoff::kScratchDoubleReg2, kScratchRegister); } void LiftoffAssembler::emit_i8x16_splat(LiftoffRegister dst, LiftoffRegister src) { - Movd(dst.fp(), src.gp()); - Pxor(kScratchDoubleReg, kScratchDoubleReg); - Pshufb(dst.fp(), kScratchDoubleReg); + I8x16Splat(dst.fp(), src.gp(), kScratchDoubleReg); } void LiftoffAssembler::emit_i16x8_splat(LiftoffRegister dst, LiftoffRegister src) { - Movd(dst.fp(), src.gp()); - Pshuflw(dst.fp(), dst.fp(), static_cast<uint8_t>(0)); - Pshufd(dst.fp(), dst.fp(), static_cast<uint8_t>(0)); + I16x8Splat(dst.fp(), src.gp()); } void LiftoffAssembler::emit_i32x4_splat(LiftoffRegister dst, @@ -2927,89 +2893,37 @@ void LiftoffAssembler::emit_i8x16_bitmask(LiftoffRegister dst, void LiftoffAssembler::emit_i8x16_shl(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { - static constexpr RegClass tmp_simd_rc = reg_class_for(kS128); - LiftoffRegister tmp_simd = - GetUnusedRegister(tmp_simd_rc, LiftoffRegList::ForRegs(dst, lhs)); - // Mask off the unwanted bits before word-shifting. - Pcmpeqw(kScratchDoubleReg, kScratchDoubleReg); - movq(kScratchRegister, rhs.gp()); - andq(kScratchRegister, Immediate(7)); - addq(kScratchRegister, Immediate(8)); - Movq(tmp_simd.fp(), kScratchRegister); - Psrlw(kScratchDoubleReg, tmp_simd.fp()); - Packuswb(kScratchDoubleReg, kScratchDoubleReg); - - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpand(dst.fp(), lhs.fp(), kScratchDoubleReg); - } else { - if (dst.fp() != lhs.fp()) movaps(dst.fp(), lhs.fp()); - andps(dst.fp(), kScratchDoubleReg); - } - subq(kScratchRegister, Immediate(8)); - Movq(tmp_simd.fp(), kScratchRegister); - Psllw(dst.fp(), tmp_simd.fp()); + I8x16Shl(dst.fp(), lhs.fp(), rhs.gp(), kScratchRegister, kScratchDoubleReg, + liftoff::kScratchDoubleReg2); } void LiftoffAssembler::emit_i8x16_shli(LiftoffRegister dst, LiftoffRegister lhs, int32_t rhs) { - byte shift = static_cast<byte>(rhs & 0x7); - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpsllw(dst.fp(), lhs.fp(), shift); - } else { - if (dst.fp() != lhs.fp()) movaps(dst.fp(), lhs.fp()); - psllw(dst.fp(), shift); - } - - uint8_t bmask = static_cast<uint8_t>(0xff << shift); - uint32_t mask = bmask << 24 | bmask << 16 | bmask << 8 | bmask; - movl(kScratchRegister, Immediate(mask)); - Movd(kScratchDoubleReg, kScratchRegister); - Pshufd(kScratchDoubleReg, kScratchDoubleReg, uint8_t{0}); - Pand(dst.fp(), kScratchDoubleReg); + I8x16Shl(dst.fp(), lhs.fp(), rhs, kScratchRegister, kScratchDoubleReg); } void LiftoffAssembler::emit_i8x16_shr_s(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { - liftoff::EmitI8x16Shr</*is_signed=*/true>(this, dst, lhs, rhs); + I8x16ShrS(dst.fp(), lhs.fp(), rhs.gp(), kScratchRegister, kScratchDoubleReg, + liftoff::kScratchDoubleReg2); } void LiftoffAssembler::emit_i8x16_shri_s(LiftoffRegister dst, LiftoffRegister lhs, int32_t rhs) { - Punpckhbw(kScratchDoubleReg, lhs.fp()); - Punpcklbw(dst.fp(), lhs.fp()); - uint8_t shift = (rhs & 7) + 8; - Psraw(kScratchDoubleReg, shift); - Psraw(dst.fp(), shift); - Packsswb(dst.fp(), kScratchDoubleReg); + I8x16ShrS(dst.fp(), lhs.fp(), rhs, kScratchDoubleReg); } void LiftoffAssembler::emit_i8x16_shr_u(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { - liftoff::EmitI8x16Shr</*is_signed=*/false>(this, dst, lhs, rhs); + I8x16ShrU(dst.fp(), lhs.fp(), rhs.gp(), kScratchRegister, kScratchDoubleReg, + liftoff::kScratchDoubleReg2); } void LiftoffAssembler::emit_i8x16_shri_u(LiftoffRegister dst, LiftoffRegister lhs, int32_t rhs) { - // Perform 16-bit shift, then mask away high bits. - uint8_t shift = rhs & 7; // i.InputInt3(1); - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpsrlw(dst.fp(), lhs.fp(), byte{shift}); - } else if (dst != lhs) { - Movaps(dst.fp(), lhs.fp()); - psrlw(dst.fp(), byte{shift}); - } - - uint8_t bmask = 0xff >> shift; - uint32_t mask = bmask << 24 | bmask << 16 | bmask << 8 | bmask; - movl(kScratchRegister, Immediate(mask)); - Movd(kScratchDoubleReg, kScratchRegister); - Pshufd(kScratchDoubleReg, kScratchDoubleReg, byte{0}); - Pand(dst.fp(), kScratchDoubleReg); + I8x16ShrU(dst.fp(), lhs.fp(), rhs, kScratchRegister, kScratchDoubleReg); } void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, @@ -3220,14 +3134,13 @@ void LiftoffAssembler::emit_i16x8_max_u(LiftoffRegister dst, void LiftoffAssembler::emit_i16x8_extadd_pairwise_i8x16_s(LiftoffRegister dst, LiftoffRegister src) { - I16x8ExtAddPairwiseI8x16S(dst.fp(), src.fp()); + I16x8ExtAddPairwiseI8x16S(dst.fp(), src.fp(), kScratchDoubleReg, + kScratchRegister); } void LiftoffAssembler::emit_i16x8_extadd_pairwise_i8x16_u(LiftoffRegister dst, LiftoffRegister src) { - Operand op = ExternalReferenceAsOperand( - ExternalReference::address_of_wasm_i8x16_splat_0x01()); - Pmaddubsw(dst.fp(), src.fp(), op); + I16x8ExtAddPairwiseI8x16U(dst.fp(), src.fp(), kScratchRegister); } void LiftoffAssembler::emit_i16x8_extmul_low_i8x16_s(LiftoffRegister dst, @@ -3259,7 +3172,7 @@ void LiftoffAssembler::emit_i16x8_extmul_high_i8x16_u(LiftoffRegister dst, void LiftoffAssembler::emit_i16x8_q15mulr_sat_s(LiftoffRegister dst, LiftoffRegister src1, LiftoffRegister src2) { - I16x8Q15MulRSatS(dst.fp(), src1.fp(), src2.fp()); + I16x8Q15MulRSatS(dst.fp(), src1.fp(), src2.fp(), kScratchDoubleReg); } void LiftoffAssembler::emit_i32x4_neg(LiftoffRegister dst, @@ -3376,14 +3289,12 @@ void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, void LiftoffAssembler::emit_i32x4_extadd_pairwise_i16x8_s(LiftoffRegister dst, LiftoffRegister src) { - Operand op = ExternalReferenceAsOperand( - ExternalReference::address_of_wasm_i16x8_splat_0x0001()); - Pmaddwd(dst.fp(), src.fp(), op); + I32x4ExtAddPairwiseI16x8S(dst.fp(), src.fp(), kScratchRegister); } void LiftoffAssembler::emit_i32x4_extadd_pairwise_i16x8_u(LiftoffRegister dst, LiftoffRegister src) { - I32x4ExtAddPairwiseI16x8U(dst.fp(), src.fp()); + I32x4ExtAddPairwiseI16x8U(dst.fp(), src.fp(), kScratchDoubleReg); } namespace liftoff { @@ -3574,28 +3485,12 @@ void LiftoffAssembler::emit_i64x2_uconvert_i32x4_high(LiftoffRegister dst, void LiftoffAssembler::emit_f32x4_abs(LiftoffRegister dst, LiftoffRegister src) { - if (dst.fp() == src.fp()) { - Pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); - Psrld(kScratchDoubleReg, static_cast<byte>(1)); - Andps(dst.fp(), kScratchDoubleReg); - } else { - Pcmpeqd(dst.fp(), dst.fp()); - Psrld(dst.fp(), static_cast<byte>(1)); - Andps(dst.fp(), src.fp()); - } + Absps(dst.fp(), src.fp()); } void LiftoffAssembler::emit_f32x4_neg(LiftoffRegister dst, LiftoffRegister src) { - if (dst.fp() == src.fp()) { - Pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); - Pslld(kScratchDoubleReg, byte{31}); - Xorps(dst.fp(), kScratchDoubleReg); - } else { - Pcmpeqd(dst.fp(), dst.fp()); - Pslld(dst.fp(), byte{31}); - Xorps(dst.fp(), src.fp()); - } + Negps(dst.fp(), src.fp()); } void LiftoffAssembler::emit_f32x4_sqrt(LiftoffRegister dst, @@ -3730,28 +3625,12 @@ void LiftoffAssembler::emit_f32x4_pmax(LiftoffRegister dst, LiftoffRegister lhs, void LiftoffAssembler::emit_f64x2_abs(LiftoffRegister dst, LiftoffRegister src) { - if (dst.fp() == src.fp()) { - Pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); - Psrlq(kScratchDoubleReg, byte{1}); - Andpd(dst.fp(), kScratchDoubleReg); - } else { - Pcmpeqd(dst.fp(), dst.fp()); - Psrlq(dst.fp(), byte{1}); - Andpd(dst.fp(), src.fp()); - } + Abspd(dst.fp(), src.fp()); } void LiftoffAssembler::emit_f64x2_neg(LiftoffRegister dst, LiftoffRegister src) { - if (dst.fp() == src.fp()) { - Pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); - Psllq(kScratchDoubleReg, static_cast<byte>(63)); - Xorpd(dst.fp(), kScratchDoubleReg); - } else { - Pcmpeqd(dst.fp(), dst.fp()); - Psllq(dst.fp(), static_cast<byte>(63)); - Xorpd(dst.fp(), src.fp()); - } + Negpd(dst.fp(), src.fp()); } void LiftoffAssembler::emit_f64x2_sqrt(LiftoffRegister dst, @@ -3842,7 +3721,7 @@ void LiftoffAssembler::emit_f64x2_convert_low_i32x4_s(LiftoffRegister dst, void LiftoffAssembler::emit_f64x2_convert_low_i32x4_u(LiftoffRegister dst, LiftoffRegister src) { - F64x2ConvertLowI32x4U(dst.fp(), src.fp()); + F64x2ConvertLowI32x4U(dst.fp(), src.fp(), kScratchRegister); } void LiftoffAssembler::emit_f64x2_promote_low_f32x4(LiftoffRegister dst, @@ -3852,26 +3731,7 @@ void LiftoffAssembler::emit_f64x2_promote_low_f32x4(LiftoffRegister dst, void LiftoffAssembler::emit_i32x4_sconvert_f32x4(LiftoffRegister dst, LiftoffRegister src) { - // NAN->0 - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vcmpeqps(kScratchDoubleReg, src.fp(), src.fp()); - vpand(dst.fp(), src.fp(), kScratchDoubleReg); - } else { - movaps(kScratchDoubleReg, src.fp()); - cmpeqps(kScratchDoubleReg, kScratchDoubleReg); - if (dst.fp() != src.fp()) movaps(dst.fp(), src.fp()); - andps(dst.fp(), kScratchDoubleReg); - } - // Set top bit if >= 0 (but not -0.0!). - Pxor(kScratchDoubleReg, dst.fp()); - // Convert to int. - Cvttps2dq(dst.fp(), dst.fp()); - // Set top bit if >=0 is now < 0. - Pand(kScratchDoubleReg, dst.fp()); - Psrad(kScratchDoubleReg, byte{31}); - // Set positive overflow lanes to 0x7FFFFFFF. - Pxor(dst.fp(), kScratchDoubleReg); + I32x4SConvertF32x4(dst.fp(), src.fp(), kScratchDoubleReg, kScratchRegister); } void LiftoffAssembler::emit_i32x4_uconvert_f32x4(LiftoffRegister dst, @@ -4012,12 +3872,14 @@ void LiftoffAssembler::emit_i32x4_uconvert_i16x8_high(LiftoffRegister dst, void LiftoffAssembler::emit_i32x4_trunc_sat_f64x2_s_zero(LiftoffRegister dst, LiftoffRegister src) { - I32x4TruncSatF64x2SZero(dst.fp(), src.fp()); + I32x4TruncSatF64x2SZero(dst.fp(), src.fp(), kScratchDoubleReg, + kScratchRegister); } void LiftoffAssembler::emit_i32x4_trunc_sat_f64x2_u_zero(LiftoffRegister dst, LiftoffRegister src) { - I32x4TruncSatF64x2UZero(dst.fp(), src.fp()); + I32x4TruncSatF64x2UZero(dst.fp(), src.fp(), kScratchDoubleReg, + kScratchRegister); } void LiftoffAssembler::emit_s128_and_not(LiftoffRegister dst, @@ -4322,11 +4184,7 @@ void LiftoffAssembler::CallIndirect(const ValueKindSig* sig, popq(kScratchRegister); target = kScratchRegister; } - if (FLAG_untrusted_code_mitigations) { - RetpolineCall(target); - } else { - call(target); - } + call(target); } void LiftoffAssembler::TailCallIndirect(Register target) { @@ -4334,11 +4192,7 @@ void LiftoffAssembler::TailCallIndirect(Register target) { popq(kScratchRegister); target = kScratchRegister; } - if (FLAG_untrusted_code_mitigations) { - RetpolineJump(target); - } else { - jmp(target); - } + jmp(target); } void LiftoffAssembler::CallRuntimeStub(WasmCode::RuntimeStubId sid) { @@ -4376,19 +4230,19 @@ void LiftoffAssembler::emit_set_if_nan(Register dst, DoubleRegister src, bind(&ret); } -void LiftoffAssembler::emit_s128_set_if_nan(Register dst, DoubleRegister src, +void LiftoffAssembler::emit_s128_set_if_nan(Register dst, LiftoffRegister src, Register tmp_gp, - DoubleRegister tmp_fp, + LiftoffRegister tmp_s128, ValueKind lane_kind) { if (lane_kind == kF32) { - movaps(tmp_fp, src); - cmpunordps(tmp_fp, tmp_fp); + movaps(tmp_s128.fp(), src.fp()); + cmpunordps(tmp_s128.fp(), tmp_s128.fp()); } else { DCHECK_EQ(lane_kind, kF64); - movapd(tmp_fp, src); - cmpunordpd(tmp_fp, tmp_fp); + movapd(tmp_s128.fp(), src.fp()); + cmpunordpd(tmp_s128.fp(), tmp_s128.fp()); } - pmovmskb(tmp_gp, tmp_fp); + pmovmskb(tmp_gp, tmp_s128.fp()); orl(Operand(dst, 0), tmp_gp); } |