diff options
Diffstat (limited to 'deps/v8/src/codegen')
90 files changed, 4264 insertions, 2325 deletions
diff --git a/deps/v8/src/codegen/OWNERS b/deps/v8/src/codegen/OWNERS index 641ed9c403..364d34fb09 100644 --- a/deps/v8/src/codegen/OWNERS +++ b/deps/v8/src/codegen/OWNERS @@ -1,10 +1,8 @@ bbudge@chromium.org -bmeurer@chromium.org clemensb@chromium.org delphick@chromium.org gdeepti@chromium.org ishell@chromium.org -jarin@chromium.org jgruber@chromium.org jkummerow@chromium.org leszeks@chromium.org @@ -12,7 +10,7 @@ mslekova@chromium.org mvstanton@chromium.org mythria@chromium.org neis@chromium.org +nicohartmann@chromium.org rmcilroy@chromium.org -sigurds@chromium.org solanes@chromium.org zhin@chromium.org diff --git a/deps/v8/src/codegen/aligned-slot-allocator.cc b/deps/v8/src/codegen/aligned-slot-allocator.cc new file mode 100644 index 0000000000..9e7ab09c81 --- /dev/null +++ b/deps/v8/src/codegen/aligned-slot-allocator.cc @@ -0,0 +1,125 @@ +// Copyright 2020 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/codegen/aligned-slot-allocator.h" + +#include "src/base/bits.h" +#include "src/base/logging.h" + +namespace v8 { +namespace internal { + +int AlignedSlotAllocator::NextSlot(int n) const { + DCHECK(n == 1 || n == 2 || n == 4); + if (n <= 1 && IsValid(next1_)) return next1_; + if (n <= 2 && IsValid(next2_)) return next2_; + DCHECK(IsValid(next4_)); + return next4_; +} + +int AlignedSlotAllocator::Allocate(int n) { + DCHECK(n == 1 || n == 2 || n == 4); + // Check invariants. + DCHECK_EQ(0, next4_ & 3); + DCHECK_IMPLIES(IsValid(next2_), (next2_ & 1) == 0); + + // The sentinel value kInvalidSlot is used to indicate no slot. + // next1_ is the index of the 1 slot fragment, or kInvalidSlot. + // next2_ is the 2-aligned index of the 2 slot fragment, or kInvalidSlot. + // next4_ is the 4-aligned index of the next 4 slot group. It is always valid. + // In order to ensure we only have a single 1- or 2-slot fragment, we greedily + // use any fragment that satisfies the request. + int result = kInvalidSlot; + switch (n) { + case 1: { + if (IsValid(next1_)) { + result = next1_; + next1_ = kInvalidSlot; + } else if (IsValid(next2_)) { + result = next2_; + next1_ = result + 1; + next2_ = kInvalidSlot; + } else { + result = next4_; + next1_ = result + 1; + next2_ = result + 2; + next4_ += 4; + } + break; + } + case 2: { + if (IsValid(next2_)) { + result = next2_; + next2_ = kInvalidSlot; + } else { + result = next4_; + next2_ = result + 2; + next4_ += 4; + } + break; + } + case 4: { + result = next4_; + next4_ += 4; + break; + } + default: + UNREACHABLE(); + break; + } + DCHECK(IsValid(result)); + size_ = std::max(size_, result + n); + return result; +} + +int AlignedSlotAllocator::AllocateUnaligned(int n) { + DCHECK_GE(n, 0); + // Check invariants. + DCHECK_EQ(0, next4_ & 3); + DCHECK_IMPLIES(IsValid(next2_), (next2_ & 1) == 0); + + // Reserve |n| slots at |size_|, invalidate fragments below the new |size_|, + // and add any new fragments beyond the new |size_|. + int result = size_; + size_ += n; + switch (size_ & 3) { + case 0: { + next1_ = next2_ = kInvalidSlot; + next4_ = size_; + break; + } + case 1: { + next1_ = size_; + next2_ = size_ + 1; + next4_ = size_ + 3; + break; + } + case 2: { + next1_ = kInvalidSlot; + next2_ = size_; + next4_ = size_ + 2; + break; + } + case 3: { + next1_ = size_; + next2_ = kInvalidSlot; + next4_ = size_ + 1; + break; + } + } + return result; +} + +int AlignedSlotAllocator::Align(int n) { + DCHECK(base::bits::IsPowerOfTwo(n)); + DCHECK_LE(n, 4); + int mask = n - 1; + int misalignment = size_ & mask; + int padding = (n - misalignment) & mask; + AllocateUnaligned(padding); + return padding; +} + +} // namespace internal +} // namespace v8 diff --git a/deps/v8/src/codegen/aligned-slot-allocator.h b/deps/v8/src/codegen/aligned-slot-allocator.h new file mode 100644 index 0000000000..1abb711713 --- /dev/null +++ b/deps/v8/src/codegen/aligned-slot-allocator.h @@ -0,0 +1,71 @@ +// Copyright 2020 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_CODEGEN_ALIGNED_SLOT_ALLOCATOR_H_ +#define V8_CODEGEN_ALIGNED_SLOT_ALLOCATOR_H_ + +#include "src/base/macros.h" +#include "src/base/platform/platform.h" +#include "src/common/globals.h" + +namespace v8 { +namespace internal { + +// An aligned slot allocator. Allocates groups of 1, 2, or 4 slots such that the +// first slot of the group is aligned to the group size. The allocator can also +// allocate unaligned groups of arbitrary size, and an align the number of slots +// to 1, 2, or 4 slots. The allocator tries to be as thrifty as possible by +// reusing alignment padding slots in subsequent smaller slot allocations. +class V8_EXPORT_PRIVATE AlignedSlotAllocator { + public: + // Slots are always multiples of pointer-sized units. + static constexpr int kSlotSize = kSystemPointerSize; + + static int NumSlotsForWidth(int bytes) { + DCHECK_GT(bytes, 0); + return (bytes + kSlotSize - 1) / kSlotSize; + } + + AlignedSlotAllocator() = default; + + // Allocates |n| slots, where |n| must be 1, 2, or 4. Padding slots may be + // inserted for alignment. + // Returns the starting index of the slots, which is evenly divisible by |n|. + int Allocate(int n); + + // Gets the starting index of the slots that would be returned by Allocate(n). + int NextSlot(int n) const; + + // Allocates the given number of slots at the current end of the slot area, + // and returns the starting index of the slots. This resets any fragment + // slots, so subsequent allocations will be after the end of this one. + // AllocateUnaligned(0) can be used to partition the slot area, for example + // to make sure tagged values follow untagged values on a Frame. + int AllocateUnaligned(int n); + + // Aligns the slot area so that future allocations begin at the alignment. + // Returns the number of slots needed to align the slot area. + int Align(int n); + + // Returns the size of the slot area, in slots. This will be greater than any + // already allocated slot index. + int Size() const { return size_; } + + private: + static constexpr int kInvalidSlot = -1; + + static bool IsValid(int slot) { return slot > kInvalidSlot; } + + int next1_ = kInvalidSlot; + int next2_ = kInvalidSlot; + int next4_ = 0; + int size_ = 0; + + DISALLOW_NEW_AND_DELETE() +}; + +} // namespace internal +} // namespace v8 + +#endif // V8_CODEGEN_ALIGNED_SLOT_ALLOCATOR_H_ diff --git a/deps/v8/src/codegen/arm/assembler-arm-inl.h b/deps/v8/src/codegen/arm/assembler-arm-inl.h index 9dadad96ac..7035fa2492 100644 --- a/deps/v8/src/codegen/arm/assembler-arm-inl.h +++ b/deps/v8/src/codegen/arm/assembler-arm-inl.h @@ -49,8 +49,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return true; } -bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(NEON); } - int DoubleRegister::SupportedRegisterCount() { return CpuFeatures::IsSupported(VFP32DREGS) ? 32 : 16; } diff --git a/deps/v8/src/codegen/arm/assembler-arm.cc b/deps/v8/src/codegen/arm/assembler-arm.cc index 6af924fa47..17a20a6f97 100644 --- a/deps/v8/src/codegen/arm/assembler-arm.cc +++ b/deps/v8/src/codegen/arm/assembler-arm.cc @@ -198,6 +198,8 @@ static constexpr unsigned CpuFeaturesFromCompiler() { #endif } +bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(NEON); } + void CpuFeatures::ProbeImpl(bool cross_compile) { dcache_line_size_ = 64; @@ -239,9 +241,9 @@ void CpuFeatures::ProbeImpl(bool cross_compile) { // Additional tuning options. // ARM Cortex-A9 and Cortex-A5 have 32 byte cachelines. - if (cpu.implementer() == base::CPU::ARM && - (cpu.part() == base::CPU::ARM_CORTEX_A5 || - cpu.part() == base::CPU::ARM_CORTEX_A9)) { + if (cpu.implementer() == base::CPU::kArm && + (cpu.part() == base::CPU::kArmCortexA5 || + cpu.part() == base::CPU::kArmCortexA9)) { dcache_line_size_ = 32; } #endif @@ -4248,6 +4250,15 @@ void Assembler::vorr(QwNeonRegister dst, QwNeonRegister src1, src2.code())); } +void Assembler::vorn(QwNeonRegister dst, QwNeonRegister src1, + QwNeonRegister src2) { + // Qd = vorn(Qn, Qm) SIMD OR NOT. + // Instruction details available in ARM DDI 0406C.d, A8.8.359. + DCHECK(IsEnabled(NEON)); + emit(EncodeNeonBinaryBitwiseOp(VORN, NEON_Q, dst.code(), src1.code(), + src2.code())); +} + enum FPBinOp { VADDF, VSUBF, diff --git a/deps/v8/src/codegen/arm/assembler-arm.h b/deps/v8/src/codegen/arm/assembler-arm.h index 456ac03f92..e0490a6853 100644 --- a/deps/v8/src/codegen/arm/assembler-arm.h +++ b/deps/v8/src/codegen/arm/assembler-arm.h @@ -887,6 +887,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { void veor(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); void vbsl(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); void vorr(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); + void vorn(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); void vadd(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); void vadd(NeonSize size, QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); @@ -1381,6 +1382,20 @@ class V8_EXPORT_PRIVATE V8_NODISCARD UseScratchRegisterScope { bool CanAcquire() const { return *assembler_->GetScratchRegisterList() != 0; } bool CanAcquireD() const { return CanAcquireVfp<DwVfpRegister>(); } + void Include(const Register& reg1, const Register& reg2 = no_reg) { + RegList* available = assembler_->GetScratchRegisterList(); + DCHECK_NOT_NULL(available); + DCHECK_EQ((*available) & (reg1.bit() | reg2.bit()), 0); + *available |= reg1.bit() | reg2.bit(); + } + void Exclude(const Register& reg1, const Register& reg2 = no_reg) { + RegList* available = assembler_->GetScratchRegisterList(); + DCHECK_NOT_NULL(available); + DCHECK_EQ((*available) & (reg1.bit() | reg2.bit()), + reg1.bit() | reg2.bit()); + *available &= ~(reg1.bit() | reg2.bit()); + } + private: friend class Assembler; friend class TurboAssembler; diff --git a/deps/v8/src/codegen/arm/interface-descriptors-arm.cc b/deps/v8/src/codegen/arm/interface-descriptors-arm.cc index 25063b2a32..53992227ab 100644 --- a/deps/v8/src/codegen/arm/interface-descriptors-arm.cc +++ b/deps/v8/src/codegen/arm/interface-descriptors-arm.cc @@ -86,14 +86,8 @@ const Register ApiGetterDescriptor::CallbackRegister() { return r3; } const Register GrowArrayElementsDescriptor::ObjectRegister() { return r0; } const Register GrowArrayElementsDescriptor::KeyRegister() { return r3; } -const Register BaselineLeaveFrameDescriptor::ParamsSizeRegister() { - // TODO(v8:11421): Implement on this platform. - UNREACHABLE(); -} -const Register BaselineLeaveFrameDescriptor::WeightRegister() { - // TODO(v8:11421): Implement on this platform. - UNREACHABLE(); -} +const Register BaselineLeaveFrameDescriptor::ParamsSizeRegister() { return r3; } +const Register BaselineLeaveFrameDescriptor::WeightRegister() { return r4; } // static const Register TypeConversionDescriptor::ArgumentRegister() { return r0; } @@ -220,8 +214,11 @@ void CompareDescriptor::InitializePlatformSpecific( void Compare_BaselineDescriptor::InitializePlatformSpecific( CallInterfaceDescriptorData* data) { - // TODO(v8:11421): Implement on this platform. - InitializePlatformUnimplemented(data, kParameterCount); + // r1: left operand + // r0: right operand + // r2: feedback slot + Register registers[] = {r1, r0, r2}; + data->InitializePlatformSpecific(arraysize(registers), registers); } void BinaryOpDescriptor::InitializePlatformSpecific( @@ -232,8 +229,11 @@ void BinaryOpDescriptor::InitializePlatformSpecific( void BinaryOp_BaselineDescriptor::InitializePlatformSpecific( CallInterfaceDescriptorData* data) { - // TODO(v8:11421): Implement on this platform. - InitializePlatformUnimplemented(data, kParameterCount); + // r1: left operand + // r0: right operand + // r2: feedback slot + Register registers[] = {r1, r0, r2}; + data->InitializePlatformSpecific(arraysize(registers), registers); } void ApiCallbackDescriptor::InitializePlatformSpecific( diff --git a/deps/v8/src/codegen/arm/macro-assembler-arm.cc b/deps/v8/src/codegen/arm/macro-assembler-arm.cc index f1831aaea3..f83eee4a91 100644 --- a/deps/v8/src/codegen/arm/macro-assembler-arm.cc +++ b/deps/v8/src/codegen/arm/macro-assembler-arm.cc @@ -25,7 +25,10 @@ #include "src/runtime/runtime.h" #include "src/snapshot/embedded/embedded-data.h" #include "src/snapshot/snapshot.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY // Satisfy cpplint check, but don't include platform-specific header. It is // included recursively via macro-assembler.h. @@ -302,6 +305,18 @@ void TurboAssembler::CallBuiltinByIndex(Register builtin_index) { Call(builtin_index); } +void TurboAssembler::LoadEntryFromBuiltinIndex(Builtins::Name builtin_index, + Register destination) { + ldr(destination, EntryFromBuiltinIndexAsOperand(builtin_index)); +} + +MemOperand TurboAssembler::EntryFromBuiltinIndexAsOperand( + Builtins::Name builtin_index) { + DCHECK(root_array_available()); + return MemOperand(kRootRegister, + IsolateData::builtin_entry_slot_offset(builtin_index)); +} + void TurboAssembler::CallBuiltin(int builtin_index, Condition cond) { DCHECK(Builtins::IsBuiltinId(builtin_index)); RecordCommentForOffHeapTrampoline(builtin_index); @@ -1344,8 +1359,11 @@ void TurboAssembler::EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) { // r0-r3: preserved UseScratchRegisterScope temps(this); - Register scratch = temps.Acquire(); - mov(scratch, Operand(StackFrame::TypeToMarker(type))); + Register scratch = no_reg; + if (!StackFrame::IsJavaScript(type)) { + scratch = temps.Acquire(); + mov(scratch, Operand(StackFrame::TypeToMarker(type))); + } PushCommonFrame(scratch); } @@ -1386,6 +1404,7 @@ void TurboAssembler::AllocateStackSpace(Register bytes_scratch) { } void TurboAssembler::AllocateStackSpace(int bytes) { + DCHECK_GE(bytes, 0); UseScratchRegisterScope temps(this); DwVfpRegister scratch = no_dreg; while (bytes > kStackPageSize) { @@ -1396,6 +1415,7 @@ void TurboAssembler::AllocateStackSpace(int bytes) { vldr(scratch, MemOperand(sp)); bytes -= kStackPageSize; } + if (bytes == 0) return; sub(sp, sp, Operand(bytes)); } #endif @@ -1913,8 +1933,13 @@ void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone, AllocateStackSpace(kDoubleSize); // Put input on stack. vstr(double_input, MemOperand(sp, 0)); +#if V8_ENABLE_WEBASSEMBLY if (stub_mode == StubCallMode::kCallWasmRuntimeStub) { Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL); +#else + // For balance. + if (false) { +#endif // V8_ENABLE_WEBASSEMBLY } else if (options().inline_offheap_trampolines) { CallBuiltin(Builtins::kDoubleToI); } else { @@ -2070,10 +2095,10 @@ void TurboAssembler::LoadMap(Register destination, Register object) { } void MacroAssembler::LoadGlobalProxy(Register dst) { - LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst); + LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX); } -void MacroAssembler::LoadNativeContextSlot(int index, Register dst) { +void MacroAssembler::LoadNativeContextSlot(Register dst, int index) { LoadMap(dst, cp); ldr(dst, FieldMemOperand( dst, Map::kConstructorOrBackPointerOrNativeContextOffset)); @@ -2650,6 +2675,16 @@ void TurboAssembler::I64x2Eq(QwNeonRegister dst, QwNeonRegister src1, vand(dst, dst, scratch); } +void TurboAssembler::I64x2Ne(QwNeonRegister dst, QwNeonRegister src1, + QwNeonRegister src2) { + UseScratchRegisterScope temps(this); + Simd128Register tmp = temps.AcquireQ(); + vceq(Neon32, dst, src1, src2); + vrev64(Neon32, tmp, dst); + vmvn(dst, dst); + vorn(dst, dst, tmp); +} + void TurboAssembler::I64x2GtS(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2) { vqsub(NeonS64, dst, src2, src1); @@ -2663,7 +2698,7 @@ void TurboAssembler::I64x2GeS(QwNeonRegister dst, QwNeonRegister src1, vmvn(dst, dst); } -void TurboAssembler::V64x2AllTrue(Register dst, QwNeonRegister src) { +void TurboAssembler::I64x2AllTrue(Register dst, QwNeonRegister src) { UseScratchRegisterScope temps(this); QwNeonRegister tmp = temps.AcquireQ(); // src = | a | b | c | d | @@ -2686,6 +2721,49 @@ void TurboAssembler::V64x2AllTrue(Register dst, QwNeonRegister src) { // = defintion of i64x2.all_true. } +void TurboAssembler::I64x2Abs(QwNeonRegister dst, QwNeonRegister src) { + UseScratchRegisterScope temps(this); + Simd128Register tmp = temps.AcquireQ(); + vshr(NeonS64, tmp, src, 63); + veor(dst, src, tmp); + vsub(Neon64, dst, dst, tmp); +} + +namespace { +using AssemblerFunc = void (Assembler::*)(DwVfpRegister, SwVfpRegister, + VFPConversionMode, const Condition); +// Helper function for f64x2 convert low instructions. +// This ensures that we do not overwrite src, if dst == src. +void F64x2ConvertLowHelper(Assembler* assm, QwNeonRegister dst, + QwNeonRegister src, AssemblerFunc convert_fn) { + LowDwVfpRegister src_d = LowDwVfpRegister::from_code(src.low().code()); + UseScratchRegisterScope temps(assm); + if (dst == src) { + LowDwVfpRegister tmp = temps.AcquireLowD(); + assm->vmov(tmp, src_d); + src_d = tmp; + } + // Default arguments are not part of the function type + (assm->*convert_fn)(dst.low(), src_d.low(), kDefaultRoundToZero, al); + (assm->*convert_fn)(dst.high(), src_d.high(), kDefaultRoundToZero, al); +} +} // namespace + +void TurboAssembler::F64x2ConvertLowI32x4S(QwNeonRegister dst, + QwNeonRegister src) { + F64x2ConvertLowHelper(this, dst, src, &Assembler::vcvt_f64_s32); +} + +void TurboAssembler::F64x2ConvertLowI32x4U(QwNeonRegister dst, + QwNeonRegister src) { + F64x2ConvertLowHelper(this, dst, src, &Assembler::vcvt_f64_u32); +} + +void TurboAssembler::F64x2PromoteLowF32x4(QwNeonRegister dst, + QwNeonRegister src) { + F64x2ConvertLowHelper(this, dst, src, &Assembler::vcvt_f64_f32); +} + } // namespace internal } // namespace v8 diff --git a/deps/v8/src/codegen/arm/macro-assembler-arm.h b/deps/v8/src/codegen/arm/macro-assembler-arm.h index 54c3e6c941..e622d4aa17 100644 --- a/deps/v8/src/codegen/arm/macro-assembler-arm.h +++ b/deps/v8/src/codegen/arm/macro-assembler-arm.h @@ -12,7 +12,7 @@ #include "src/codegen/arm/assembler-arm.h" #include "src/codegen/bailout-reason.h" #include "src/common/globals.h" -#include "src/objects/contexts.h" +#include "src/objects/tagged-index.h" namespace v8 { namespace internal { @@ -64,7 +64,11 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void AllocateStackSpace(int bytes); #else void AllocateStackSpace(Register bytes) { sub(sp, sp, bytes); } - void AllocateStackSpace(int bytes) { sub(sp, sp, Operand(bytes)); } + void AllocateStackSpace(int bytes) { + DCHECK_GE(bytes, 0); + if (bytes == 0) return; + sub(sp, sp, Operand(bytes)); + } #endif // Push a fixed frame, consisting of lr, fp @@ -309,6 +313,9 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { bool check_constant_pool = true); void Call(Label* target); + MemOperand EntryFromBuiltinIndexAsOperand(Builtins::Name builtin_index); + void LoadEntryFromBuiltinIndex(Builtins::Name builtin_index, + Register destination); // Load the builtin given by the Smi in |builtin_index| into the same // register. void LoadEntryFromBuiltinIndex(Register builtin_index); @@ -474,6 +481,7 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void Move(Register dst, Handle<HeapObject> value); void Move(Register dst, ExternalReference reference); void Move(Register dst, Register src, Condition cond = al); + void Move(Register dst, const MemOperand& src) { ldr(dst, src); } void Move(Register dst, const Operand& src, SBit sbit = LeaveCC, Condition cond = al) { if (!src.IsRegister() || src.rm() != dst || sbit != LeaveCC) { @@ -571,9 +579,14 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { // and be used in both TurboFan and Liftoff. void I64x2BitMask(Register dst, QwNeonRegister src); void I64x2Eq(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); + void I64x2Ne(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); void I64x2GtS(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); void I64x2GeS(QwNeonRegister dst, QwNeonRegister src1, QwNeonRegister src2); - void V64x2AllTrue(Register dst, QwNeonRegister src); + void I64x2AllTrue(Register dst, QwNeonRegister src); + void I64x2Abs(QwNeonRegister dst, QwNeonRegister src); + void F64x2ConvertLowI32x4S(QwNeonRegister dst, QwNeonRegister src); + void F64x2ConvertLowI32x4U(QwNeonRegister dst, QwNeonRegister src); + void F64x2PromoteLowF32x4(QwNeonRegister dst, QwNeonRegister src); private: // Compare single values and then load the fpscr flags to a register. @@ -668,7 +681,7 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // Load the global proxy from the current context. void LoadGlobalProxy(Register dst); - void LoadNativeContextSlot(int index, Register dst); + void LoadNativeContextSlot(Register dst, int index); // --------------------------------------------------------------------------- // JavaScript invokes diff --git a/deps/v8/src/codegen/arm/register-arm.h b/deps/v8/src/codegen/arm/register-arm.h index 6cb6c602c2..6608ad4ede 100644 --- a/deps/v8/src/codegen/arm/register-arm.h +++ b/deps/v8/src/codegen/arm/register-arm.h @@ -119,7 +119,12 @@ GENERAL_REGISTERS(DECLARE_REGISTER) #undef DECLARE_REGISTER constexpr Register no_reg = Register::no_reg(); -constexpr bool kPadArguments = false; +// Returns the number of padding slots needed for stack pointer alignment. +constexpr int ArgumentPaddingSlots(int argument_count) { + // No argument padding required. + return 0; +} + constexpr bool kSimpleFPAliasing = false; constexpr bool kSimdMaskRegisters = false; diff --git a/deps/v8/src/codegen/arm64/assembler-arm64-inl.h b/deps/v8/src/codegen/arm64/assembler-arm64-inl.h index 1027dccc47..ee64dbe1f2 100644 --- a/deps/v8/src/codegen/arm64/assembler-arm64-inl.h +++ b/deps/v8/src/codegen/arm64/assembler-arm64-inl.h @@ -19,8 +19,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return true; } -bool CpuFeatures::SupportsWasmSimd128() { return true; } - void RelocInfo::apply(intptr_t delta) { // On arm64 only internal references and immediate branches need extra work. if (RelocInfo::IsInternalReference(rmode_)) { diff --git a/deps/v8/src/codegen/arm64/assembler-arm64.cc b/deps/v8/src/codegen/arm64/assembler-arm64.cc index 441f299a17..0551877403 100644 --- a/deps/v8/src/codegen/arm64/assembler-arm64.cc +++ b/deps/v8/src/codegen/arm64/assembler-arm64.cc @@ -32,6 +32,7 @@ #include "src/base/bits.h" #include "src/base/cpu.h" +#include "src/base/small-vector.h" #include "src/codegen/arm64/assembler-arm64-inl.h" #include "src/codegen/register-configuration.h" #include "src/codegen/safepoint-table.h" @@ -82,6 +83,7 @@ constexpr unsigned CpuFeaturesFromTargetOS() { // ----------------------------------------------------------------------------- // CpuFeatures implementation. +bool CpuFeatures::SupportsWasmSimd128() { return true; } void CpuFeatures::ProbeImpl(bool cross_compile) { // Only use statically determined features for cross compile (snapshot). @@ -1428,37 +1430,6 @@ void Assembler::stlxrh(const Register& rs, const Register& rt, Emit(STLXR_h | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt)); } -void Assembler::prfm(int prfop, const MemOperand& addr) { - // Restricted support for prfm, only register offset. - // This can probably be merged with Assembler::LoadStore as we expand support. - DCHECK(addr.IsRegisterOffset()); - DCHECK(is_uint5(prfop)); - Instr memop = PRFM | prfop | RnSP(addr.base()); - - Extend ext = addr.extend(); - Shift shift = addr.shift(); - unsigned shift_amount = addr.shift_amount(); - - // LSL is encoded in the option field as UXTX. - if (shift == LSL) { - ext = UXTX; - } - - // Shifts are encoded in one bit, indicating a left shift by the memory - // access size. - DCHECK((shift_amount == 0) || - (shift_amount == static_cast<unsigned>(CalcLSDataSize(PRFM)))); - - Emit(LoadStoreRegisterOffsetFixed | memop | Rm(addr.regoffset()) | - ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0)); -} - -void Assembler::prfm(PrefetchOperation prfop, const MemOperand& addr) { - // Restricted support for prfm, only register offset. - // This can probably be merged with Assembler::LoadStore as we expand support. - prfm(static_cast<int>(prfop), addr); -} - void Assembler::NEON3DifferentL(const VRegister& vd, const VRegister& vn, const VRegister& vm, NEON3DifferentOp vop) { DCHECK(AreSameFormat(vn, vm)); @@ -4502,12 +4473,15 @@ const size_t ConstantPool::kOpportunityDistToPool32 = 64 * KB; const size_t ConstantPool::kOpportunityDistToPool64 = 64 * KB; const size_t ConstantPool::kApproxMaxEntryCount = 512; -bool Assembler::ShouldEmitVeneer(int max_reachable_pc, size_t margin) { - // Account for the branch around the veneers and the guard. - int protection_offset = 2 * kInstrSize; - return static_cast<intptr_t>(pc_offset() + margin + protection_offset + - unresolved_branches_.size() * - kMaxVeneerCodeSize) >= max_reachable_pc; +intptr_t Assembler::MaxPCOffsetAfterVeneerPoolIfEmittedNow(size_t margin) { + // Account for the branch and guard around the veneers. + static constexpr int kBranchSizeInBytes = kInstrSize; + static constexpr int kGuardSizeInBytes = kInstrSize; + const size_t max_veneer_size_in_bytes = + unresolved_branches_.size() * kVeneerCodeSize; + return static_cast<intptr_t>(pc_offset() + kBranchSizeInBytes + + kGuardSizeInBytes + max_veneer_size_in_bytes + + margin); } void Assembler::RecordVeneerPool(int location_offset, int size) { @@ -4538,51 +4512,80 @@ void Assembler::EmitVeneers(bool force_emit, bool need_protection, EmitVeneersGuard(); -#ifdef DEBUG - Label veneer_size_check; -#endif + // We only emit veneers if needed (unless emission is forced), i.e. when the + // max-reachable-pc of the branch has been exhausted by the current codegen + // state. Specifically, we emit when the max-reachable-pc of the branch <= the + // max-pc-after-veneers (over-approximated). + const intptr_t max_pc_after_veneers = + MaxPCOffsetAfterVeneerPoolIfEmittedNow(margin); + + // The `unresolved_branches_` multimap is sorted by max-reachable-pc in + // ascending order. For efficiency reasons, we want to call + // RemoveBranchFromLabelLinkChain in descending order. The actual veneers are + // then generated in ascending order. + // TODO(jgruber): This is still inefficient in multiple ways, thoughts on how + // we could improve in the future: + // - Don't erase individual elements from the multimap, erase a range instead. + // - Replace the multimap by a simpler data structure (like a plain vector or + // a circular array). + // - Refactor s.t. RemoveBranchFromLabelLinkChain does not need the linear + // lookup in the link chain. + + static constexpr int kStaticTasksSize = 16; // Arbitrary. + base::SmallVector<FarBranchInfo, kStaticTasksSize> tasks; + + { + auto it = unresolved_branches_.begin(); + while (it != unresolved_branches_.end()) { + const int max_reachable_pc = it->first; + if (!force_emit && max_reachable_pc > max_pc_after_veneers) break; + + // Found a task. We'll emit a veneer for this. + tasks.emplace_back(it->second); + auto eraser_it = it++; + unresolved_branches_.erase(eraser_it); + } + } - std::multimap<int, FarBranchInfo>::iterator it, it_to_delete; + // Update next_veneer_pool_check_ (tightly coupled with unresolved_branches_). + if (unresolved_branches_.empty()) { + next_veneer_pool_check_ = kMaxInt; + } else { + next_veneer_pool_check_ = + unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; + } - it = unresolved_branches_.begin(); - while (it != unresolved_branches_.end()) { - if (force_emit || ShouldEmitVeneer(it->first, margin)) { - Instruction* branch = InstructionAt(it->second.pc_offset_); - Label* label = it->second.label_; + // Reminder: We iterate in reverse order to avoid duplicate linked-list + // iteration in RemoveBranchFromLabelLinkChain (which starts at the target + // label, and iterates backwards through linked branch instructions). + const int tasks_size = static_cast<int>(tasks.size()); + for (int i = tasks_size - 1; i >= 0; i--) { + Instruction* branch = InstructionAt(tasks[i].pc_offset_); + Instruction* veneer = reinterpret_cast<Instruction*>( + reinterpret_cast<uintptr_t>(pc_) + i * kVeneerCodeSize); + RemoveBranchFromLabelLinkChain(branch, tasks[i].label_, veneer); + } + + // Now emit the actual veneer and patch up the incoming branch. + + for (const FarBranchInfo& info : tasks) { #ifdef DEBUG - bind(&veneer_size_check); -#endif - // Patch the branch to point to the current position, and emit a branch - // to the label. - Instruction* veneer = reinterpret_cast<Instruction*>(pc_); - RemoveBranchFromLabelLinkChain(branch, label, veneer); - branch->SetImmPCOffsetTarget(options(), veneer); - b(label); -#ifdef DEBUG - DCHECK(SizeOfCodeGeneratedSince(&veneer_size_check) <= - static_cast<uint64_t>(kMaxVeneerCodeSize)); - veneer_size_check.Unuse(); + Label veneer_size_check; + bind(&veneer_size_check); #endif - - it_to_delete = it++; - unresolved_branches_.erase(it_to_delete); - } else { - ++it; - } + Instruction* branch = InstructionAt(info.pc_offset_); + Instruction* veneer = reinterpret_cast<Instruction*>(pc_); + branch->SetImmPCOffsetTarget(options(), veneer); + b(info.label_); // This may end up pointing at yet another veneer later on. + DCHECK_EQ(SizeOfCodeGeneratedSince(&veneer_size_check), + static_cast<uint64_t>(kVeneerCodeSize)); } // Record the veneer pool size. int pool_size = static_cast<int>(SizeOfCodeGeneratedSince(&size_check)); RecordVeneerPool(veneer_pool_relocinfo_loc, pool_size); - if (unresolved_branches_.empty()) { - next_veneer_pool_check_ = kMaxInt; - } else { - next_veneer_pool_check_ = - unresolved_branches_first_limit() - kVeneerDistanceCheckMargin; - } - bind(&end); RecordComment("]"); diff --git a/deps/v8/src/codegen/arm64/assembler-arm64.h b/deps/v8/src/codegen/arm64/assembler-arm64.h index 41bdb03b4f..aa2ffb26cd 100644 --- a/deps/v8/src/codegen/arm64/assembler-arm64.h +++ b/deps/v8/src/codegen/arm64/assembler-arm64.h @@ -880,9 +880,6 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { // Store-release exclusive half-word. void stlxrh(const Register& rs, const Register& rt, const Register& rn); - void prfm(int prfop, const MemOperand& addr); - void prfm(PrefetchOperation prfop, const MemOperand& addr); - // Move instructions. The default shift of -1 indicates that the move // instruction will calculate an appropriate 16-bit immediate and left shift // that is equal to the 64-bit immediate argument. If an explicit left shift @@ -2392,18 +2389,23 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { constpool_.Check(Emission::kIfNeeded, Jump::kRequired, margin); } + // Used by veneer checks below - returns the max (= overapproximated) pc + // offset after the veneer pool, if the veneer pool were to be emitted + // immediately. + intptr_t MaxPCOffsetAfterVeneerPoolIfEmittedNow(size_t margin); // Returns true if we should emit a veneer as soon as possible for a branch // which can at most reach to specified pc. - bool ShouldEmitVeneer(int max_reachable_pc, - size_t margin = kVeneerDistanceMargin); + bool ShouldEmitVeneer(int max_reachable_pc, size_t margin) { + return max_reachable_pc < MaxPCOffsetAfterVeneerPoolIfEmittedNow(margin); + } bool ShouldEmitVeneers(size_t margin = kVeneerDistanceMargin) { return ShouldEmitVeneer(unresolved_branches_first_limit(), margin); } - // The maximum code size generated for a veneer. Currently one branch + // The code size generated for a veneer. Currently one branch // instruction. This is for code size checking purposes, and can be extended // in the future for example if we decide to add nops between the veneers. - static constexpr int kMaxVeneerCodeSize = 1 * kInstrSize; + static constexpr int kVeneerCodeSize = 1 * kInstrSize; void RecordVeneerPool(int location_offset, int size); // Emits veneers for branches that are approaching their maximum range. diff --git a/deps/v8/src/codegen/arm64/constants-arm64.h b/deps/v8/src/codegen/arm64/constants-arm64.h index 940216fc94..52790b9faf 100644 --- a/deps/v8/src/codegen/arm64/constants-arm64.h +++ b/deps/v8/src/codegen/arm64/constants-arm64.h @@ -159,9 +159,6 @@ using float16 = uint16_t; /* store second source. */ \ V_(Rs, 20, 16, Bits) /* Store-exclusive status */ \ V_(PrefetchMode, 4, 0, Bits) \ - V_(PrefetchHint, 4, 3, Bits) \ - V_(PrefetchTarget, 2, 1, Bits) \ - V_(PrefetchStream, 0, 0, Bits) \ \ /* Common bits */ \ V_(SixtyFourBits, 31, 31, Bits) \ @@ -219,7 +216,6 @@ using float16 = uint16_t; V_(LSOpc, 23, 22, Bits) \ V_(LSVector, 26, 26, Bits) \ V_(LSSize, 31, 30, Bits) \ - V_(ImmPrefetchOperation, 4, 0, Bits) \ \ /* NEON generic fields */ \ V_(NEONQ, 30, 30, Bits) \ @@ -447,27 +443,6 @@ enum SystemRegister { ImmSystemRegister_offset }; -enum PrefetchOperation { - PLDL1KEEP = 0x00, - PLDL1STRM = 0x01, - PLDL2KEEP = 0x02, - PLDL2STRM = 0x03, - PLDL3KEEP = 0x04, - PLDL3STRM = 0x05, - PLIL1KEEP = 0x08, - PLIL1STRM = 0x09, - PLIL2KEEP = 0x0a, - PLIL2STRM = 0x0b, - PLIL3KEEP = 0x0c, - PLIL3STRM = 0x0d, - PSTL1KEEP = 0x10, - PSTL1STRM = 0x11, - PSTL2KEEP = 0x12, - PSTL2STRM = 0x13, - PSTL3KEEP = 0x14, - PSTL3STRM = 0x15, -}; - // Instruction enumerations. // // These are the masks that define a class of instructions, and the list of diff --git a/deps/v8/src/codegen/arm64/macro-assembler-arm64-inl.h b/deps/v8/src/codegen/arm64/macro-assembler-arm64-inl.h index 963f862f92..6a33f864ab 100644 --- a/deps/v8/src/codegen/arm64/macro-assembler-arm64-inl.h +++ b/deps/v8/src/codegen/arm64/macro-assembler-arm64-inl.h @@ -548,7 +548,7 @@ void TurboAssembler::Fcmp(const VRegister& fn, double value) { } } -void MacroAssembler::Fcsel(const VRegister& fd, const VRegister& fn, +void TurboAssembler::Fcsel(const VRegister& fd, const VRegister& fn, const VRegister& fm, Condition cond) { DCHECK(allow_macro_instructions()); DCHECK((cond != al) && (cond != nv)); @@ -1036,6 +1036,9 @@ void TurboAssembler::Uxtw(const Register& rd, const Register& rn) { void TurboAssembler::InitializeRootRegister() { ExternalReference isolate_root = ExternalReference::isolate_root(isolate()); Mov(kRootRegister, Operand(isolate_root)); +#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE + Mov(kPointerCageBaseRegister, Operand(isolate_root)); +#endif } void MacroAssembler::SmiTag(Register dst, Register src) { diff --git a/deps/v8/src/codegen/arm64/macro-assembler-arm64.cc b/deps/v8/src/codegen/arm64/macro-assembler-arm64.cc index 7cd6027932..a3570b8035 100644 --- a/deps/v8/src/codegen/arm64/macro-assembler-arm64.cc +++ b/deps/v8/src/codegen/arm64/macro-assembler-arm64.cc @@ -12,6 +12,7 @@ #include "src/codegen/external-reference-table.h" #include "src/codegen/macro-assembler-inl.h" #include "src/codegen/register-configuration.h" +#include "src/codegen/reloc-info.h" #include "src/debug/debug.h" #include "src/deoptimizer/deoptimizer.h" #include "src/execution/frame-constants.h" @@ -22,7 +23,10 @@ #include "src/runtime/runtime.h" #include "src/snapshot/embedded/embedded-data.h" #include "src/snapshot/snapshot.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY // Satisfy cpplint check, but don't include platform-specific header. It is // included recursively via macro-assembler.h. @@ -1766,25 +1770,36 @@ void TurboAssembler::JumpHelper(int64_t offset, RelocInfo::Mode rmode, Bind(&done); } -namespace { - // The calculated offset is either: // * the 'target' input unmodified if this is a Wasm call, or +// * the offset of the target from the code range start, if this is a call to +// un-embedded builtin, or // * the offset of the target from the current PC, in instructions, for any // other type of call. -static int64_t CalculateTargetOffset(Address target, RelocInfo::Mode rmode, - byte* pc) { +int64_t TurboAssembler::CalculateTargetOffset(Address target, + RelocInfo::Mode rmode, byte* pc) { int64_t offset = static_cast<int64_t>(target); - // The target of WebAssembly calls is still an index instead of an actual - // address at this point, and needs to be encoded as-is. - if (rmode != RelocInfo::WASM_CALL && rmode != RelocInfo::WASM_STUB_CALL) { + if (rmode == RelocInfo::WASM_CALL || rmode == RelocInfo::WASM_STUB_CALL) { + // The target of WebAssembly calls is still an index instead of an actual + // address at this point, and needs to be encoded as-is. + return offset; + } + if (RelocInfo::IsRuntimeEntry(rmode)) { + // The runtime entry targets are used for generating short builtin calls + // from JIT-compiled code (it's not used during snapshot creation). + // The value is encoded as an offset from the code range (see + // Assembler::runtime_entry_at()). + // Note, that builtin-to-builitin calls use different OFF_HEAP_TARGET mode + // and therefore are encoded differently. + DCHECK_NE(options().code_range_start, 0); + offset -= static_cast<int64_t>(options().code_range_start); + } else { offset -= reinterpret_cast<int64_t>(pc); - DCHECK_EQ(offset % kInstrSize, 0); - offset = offset / static_cast<int>(kInstrSize); } + DCHECK_EQ(offset % kInstrSize, 0); + offset = offset / static_cast<int>(kInstrSize); return offset; } -} // namespace void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode, Condition cond) { @@ -1801,14 +1816,8 @@ void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode, int builtin_index = Builtins::kNoBuiltinId; if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index)) { // Inline the trampoline. - RecordCommentForOffHeapTrampoline(builtin_index); - CHECK_NE(builtin_index, Builtins::kNoBuiltinId); - UseScratchRegisterScope temps(this); - Register scratch = temps.AcquireX(); - EmbeddedData d = EmbeddedData::FromBlob(); - Address entry = d.InstructionStartOfBuiltin(builtin_index); - Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET)); - Jump(scratch, cond); + CHECK_EQ(cond, Condition::al); // Implement if necessary. + TailCallBuiltin(builtin_index); return; } } @@ -1920,12 +1929,49 @@ void TurboAssembler::CallBuiltin(int builtin_index) { DCHECK(Builtins::IsBuiltinId(builtin_index)); RecordCommentForOffHeapTrampoline(builtin_index); CHECK_NE(builtin_index, Builtins::kNoBuiltinId); - UseScratchRegisterScope temps(this); - Register scratch = temps.AcquireX(); - EmbeddedData d = EmbeddedData::FromBlob(); - Address entry = d.InstructionStartOfBuiltin(builtin_index); - Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET)); - Call(scratch); + if (options().short_builtin_calls) { + EmbeddedData d = EmbeddedData::FromBlob(isolate()); + Address entry = d.InstructionStartOfBuiltin(builtin_index); + Call(entry, RelocInfo::RUNTIME_ENTRY); + + } else { + EmbeddedData d = EmbeddedData::FromBlob(); + Address entry = d.InstructionStartOfBuiltin(builtin_index); + UseScratchRegisterScope temps(this); + Register scratch = temps.AcquireX(); + Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET)); + Call(scratch); + } + if (FLAG_code_comments) RecordComment("]"); +} + +void TurboAssembler::TailCallBuiltin(int builtin_index) { + DCHECK(Builtins::IsBuiltinId(builtin_index)); + RecordCommentForOffHeapTrampoline(builtin_index); + CHECK_NE(builtin_index, Builtins::kNoBuiltinId); + if (options().short_builtin_calls) { + EmbeddedData d = EmbeddedData::FromBlob(isolate()); + Address entry = d.InstructionStartOfBuiltin(builtin_index); + Jump(entry, RelocInfo::RUNTIME_ENTRY); + + } else { + EmbeddedData d = EmbeddedData::FromBlob(); + Address entry = d.InstructionStartOfBuiltin(builtin_index); + // The control flow integrity (CFI) feature allows us to "sign" code entry + // points as a target for calls, jumps or both. Arm64 has special + // instructions for this purpose, so-called "landing pads" (see + // TurboAssembler::CallTarget(), TurboAssembler::JumpTarget() and + // TurboAssembler::JumpOrCallTarget()). Currently, we generate "Call" + // landing pads for CPP builtins. In order to allow tail calling to those + // builtins we have to use a workaround. + // x17 is used to allow using "Call" (i.e. `bti c`) rather than "Jump" + // (i.e. `bti j`) landing pads for the tail-called code. + Register temp = x17; + + Ldr(temp, Operand(entry, RelocInfo::OFF_HEAP_TARGET)); + Jump(temp); + } + if (FLAG_code_comments) RecordComment("]"); } void TurboAssembler::LoadCodeObjectEntry(Register destination, @@ -2424,8 +2470,13 @@ void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone, } // DoubleToI preserves any registers it needs to clobber. +#if V8_ENABLE_WEBASSEMBLY if (stub_mode == StubCallMode::kCallWasmRuntimeStub) { Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL); +#else + // For balance. + if (false) { +#endif // V8_ENABLE_WEBASSEMBLY } else if (options().inline_offheap_trampolines) { CallBuiltin(Builtins::kDoubleToI); } else { @@ -2457,7 +2508,11 @@ void TurboAssembler::Prologue() { void TurboAssembler::EnterFrame(StackFrame::Type type) { UseScratchRegisterScope temps(this); - if (type == StackFrame::INTERNAL || type == StackFrame::WASM_DEBUG_BREAK) { + if (type == StackFrame::INTERNAL +#if V8_ENABLE_WEBASSEMBLY + || type == StackFrame::WASM_DEBUG_BREAK +#endif // V8_ENABLE_WEBASSEMBLY + ) { Register type_reg = temps.AcquireX(); Mov(type_reg, StackFrame::TypeToMarker(type)); Push<TurboAssembler::kSignLR>(lr, fp, type_reg, padreg); @@ -2468,6 +2523,7 @@ void TurboAssembler::EnterFrame(StackFrame::Type type) { // sp[2] : fp // sp[1] : type // sp[0] : for alignment +#if V8_ENABLE_WEBASSEMBLY } else if (type == StackFrame::WASM || type == StackFrame::WASM_COMPILE_LAZY || type == StackFrame::WASM_EXIT) { @@ -2480,6 +2536,7 @@ void TurboAssembler::EnterFrame(StackFrame::Type type) { // sp[2] : fp // sp[1] : type // sp[0] : for alignment +#endif // V8_ENABLE_WEBASSEMBLY } else if (type == StackFrame::CONSTRUCT) { Register type_reg = temps.AcquireX(); Mov(type_reg, StackFrame::TypeToMarker(type)); @@ -2628,7 +2685,7 @@ void MacroAssembler::LeaveExitFrame(bool restore_doubles, } void MacroAssembler::LoadGlobalProxy(Register dst) { - LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst); + LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX); } void MacroAssembler::LoadWeakValue(Register out, Register in, @@ -2803,14 +2860,14 @@ void TurboAssembler::DecompressTaggedPointer(const Register& destination, const MemOperand& field_operand) { RecordComment("[ DecompressTaggedPointer"); Ldr(destination.W(), field_operand); - Add(destination, kRootRegister, destination); + Add(destination, kPointerCageBaseRegister, destination); RecordComment("]"); } void TurboAssembler::DecompressTaggedPointer(const Register& destination, const Register& source) { RecordComment("[ DecompressTaggedPointer"); - Add(destination, kRootRegister, Operand(source, UXTW)); + Add(destination, kPointerCageBaseRegister, Operand(source, UXTW)); RecordComment("]"); } @@ -2818,7 +2875,7 @@ void TurboAssembler::DecompressAnyTagged(const Register& destination, const MemOperand& field_operand) { RecordComment("[ DecompressAnyTagged"); Ldr(destination.W(), field_operand); - Add(destination, kRootRegister, destination); + Add(destination, kPointerCageBaseRegister, destination); RecordComment("]"); } @@ -3117,7 +3174,7 @@ void TurboAssembler::Abort(AbortReason reason) { TmpList()->set_list(old_tmp_list); } -void MacroAssembler::LoadNativeContextSlot(int index, Register dst) { +void MacroAssembler::LoadNativeContextSlot(Register dst, int index) { LoadMap(dst, cp); LoadTaggedPointerField( dst, FieldMemOperand( @@ -3430,6 +3487,7 @@ void TurboAssembler::RestoreFPAndLR() { #endif } +#if V8_ENABLE_WEBASSEMBLY void TurboAssembler::StoreReturnAddressInWasmExitFrame(Label* return_location) { UseScratchRegisterScope temps(this); temps.Exclude(x16, x17); @@ -3440,6 +3498,7 @@ void TurboAssembler::StoreReturnAddressInWasmExitFrame(Label* return_location) { #endif Str(x17, MemOperand(fp, WasmExitFrameConstants::kCallingPCOffset)); } +#endif // V8_ENABLE_WEBASSEMBLY void TurboAssembler::I64x2BitMask(Register dst, VRegister src) { UseScratchRegisterScope scope(this); @@ -3451,7 +3510,7 @@ void TurboAssembler::I64x2BitMask(Register dst, VRegister src) { Add(dst.W(), dst.W(), Operand(tmp2.W(), LSL, 1)); } -void TurboAssembler::V64x2AllTrue(Register dst, VRegister src) { +void TurboAssembler::I64x2AllTrue(Register dst, VRegister src) { UseScratchRegisterScope scope(this); VRegister tmp = scope.AcquireV(kFormat2D); Cmeq(tmp.V2D(), src.V2D(), 0); diff --git a/deps/v8/src/codegen/arm64/macro-assembler-arm64.h b/deps/v8/src/codegen/arm64/macro-assembler-arm64.h index ef7bc15166..a749676ccc 100644 --- a/deps/v8/src/codegen/arm64/macro-assembler-arm64.h +++ b/deps/v8/src/codegen/arm64/macro-assembler-arm64.h @@ -698,6 +698,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { const Operand& operand); void Csel(const Register& rd, const Register& rn, const Operand& operand, Condition cond); + inline void Fcsel(const VRegister& fd, const VRegister& fn, + const VRegister& fm, Condition cond); // Emits a runtime assert that the stack pointer is aligned. void AssertSpAligned(); @@ -975,7 +977,16 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { Register destination); MemOperand EntryFromBuiltinIndexAsOperand(Builtins::Name builtin_index); void CallBuiltinByIndex(Register builtin_index) override; + void CallBuiltin(Builtins::Name builtin) { + // TODO(11527): drop the int overload in favour of the Builtins::Name one. + return CallBuiltin(static_cast<int>(builtin)); + } void CallBuiltin(int builtin_index); + void TailCallBuiltin(Builtins::Name builtin) { + // TODO(11527): drop the int overload in favour of the Builtins::Name one. + return TailCallBuiltin(static_cast<int>(builtin)); + } + void TailCallBuiltin(int builtin_index); void LoadCodeObjectEntry(Register destination, Register code_object) override; void CallCodeObject(Register code_object) override; @@ -1374,13 +1385,15 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { // authenticate the LR when pointer authentication is enabled. void RestoreFPAndLR(); +#if V8_ENABLE_WEBASSEMBLY void StoreReturnAddressInWasmExitFrame(Label* return_location); +#endif // V8_ENABLE_WEBASSEMBLY // Wasm SIMD helpers. These instructions don't have direct lowering to native // instructions. These helpers allow us to define the optimal code sequence, // and be used in both TurboFan and Liftoff. void I64x2BitMask(Register dst, VRegister src); - void V64x2AllTrue(Register dst, VRegister src); + void I64x2AllTrue(Register dst, VRegister src); protected: // The actual Push and Pop implementations. These don't generate any code @@ -1447,6 +1460,9 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void LoadStorePairMacro(const CPURegister& rt, const CPURegister& rt2, const MemOperand& addr, LoadStorePairOp op); + int64_t CalculateTargetOffset(Address target, RelocInfo::Mode rmode, + byte* pc); + void JumpHelper(int64_t offset, RelocInfo::Mode rmode, Condition cond = al); void CallRecordWriteStub(Register object, Operand offset, @@ -1497,8 +1513,6 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { Condition cond); inline void Extr(const Register& rd, const Register& rn, const Register& rm, unsigned lsb); - inline void Fcsel(const VRegister& fd, const VRegister& fn, - const VRegister& fm, Condition cond); void Fcvtl(const VRegister& vd, const VRegister& vn) { DCHECK(allow_macro_instructions()); fcvtl(vd, vn); @@ -2032,7 +2046,7 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // --------------------------------------------------------------------------- // Debugging. - void LoadNativeContextSlot(int index, Register dst); + void LoadNativeContextSlot(Register dst, int index); DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler); }; diff --git a/deps/v8/src/codegen/arm64/register-arm64.h b/deps/v8/src/codegen/arm64/register-arm64.h index 31620ae965..605856e51c 100644 --- a/deps/v8/src/codegen/arm64/register-arm64.h +++ b/deps/v8/src/codegen/arm64/register-arm64.h @@ -30,11 +30,21 @@ namespace internal { // x18 is the platform register and is reserved for the use of platform ABIs. // It is known to be reserved by the OS at least on Windows and iOS. -#define ALLOCATABLE_GENERAL_REGISTERS(R) \ +#define ALWAYS_ALLOCATABLE_GENERAL_REGISTERS(R) \ R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \ R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \ R(x19) R(x20) R(x21) R(x22) R(x23) R(x24) R(x25) \ - R(x27) R(x28) + R(x27) + +#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE +#define MAYBE_ALLOCATABLE_GENERAL_REGISTERS(R) +#else +#define MAYBE_ALLOCATABLE_GENERAL_REGISTERS(R) R(x28) +#endif + +#define ALLOCATABLE_GENERAL_REGISTERS(V) \ + ALWAYS_ALLOCATABLE_GENERAL_REGISTERS(V) \ + MAYBE_ALLOCATABLE_GENERAL_REGISTERS(V) #define FLOAT_REGISTERS(V) \ V(s0) V(s1) V(s2) V(s3) V(s4) V(s5) V(s6) V(s7) \ @@ -241,7 +251,14 @@ class Register : public CPURegister { ASSERT_TRIVIALLY_COPYABLE(Register); -constexpr bool kPadArguments = true; +// Stack frame alignment and padding. +constexpr int ArgumentPaddingSlots(int argument_count) { + // Stack frames are aligned to 16 bytes. + constexpr int kStackFrameAlignment = 16; + constexpr int alignment_mask = kStackFrameAlignment / kSystemPointerSize - 1; + return argument_count & alignment_mask; +} + constexpr bool kSimpleFPAliasing = true; constexpr bool kSimdMaskRegisters = false; @@ -458,6 +475,12 @@ ALIAS_REGISTER(Register, wip1, w17); // Root register. ALIAS_REGISTER(Register, kRootRegister, x26); ALIAS_REGISTER(Register, rr, x26); +// Pointer cage base register. +#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE +ALIAS_REGISTER(Register, kPointerCageBaseRegister, x28); +#else +ALIAS_REGISTER(Register, kPointerCageBaseRegister, kRootRegister); +#endif // Context pointer register. ALIAS_REGISTER(Register, cp, x27); ALIAS_REGISTER(Register, fp, x29); @@ -524,8 +547,6 @@ using Simd128Register = VRegister; // Lists of registers. class V8_EXPORT_PRIVATE CPURegList { public: - CPURegList() = default; - template <typename... CPURegisters> explicit CPURegList(CPURegister reg0, CPURegisters... regs) : list_(CPURegister::ListOf(reg0, regs...)), diff --git a/deps/v8/src/codegen/assembler.cc b/deps/v8/src/codegen/assembler.cc index 0b71701d31..95983705ab 100644 --- a/deps/v8/src/codegen/assembler.cc +++ b/deps/v8/src/codegen/assembler.cc @@ -73,6 +73,12 @@ AssemblerOptions AssemblerOptions::Default(Isolate* isolate) { DCHECK_IMPLIES(code_range.begin() != kNullAddress, !code_range.is_empty()); options.code_range_start = code_range.begin(); #endif + options.short_builtin_calls = + isolate->is_short_builtin_calls_enabled() && + !generating_embedded_builtin && + (options.code_range_start != kNullAddress) && + // Serialization of RUNTIME_ENTRY reloc infos is not supported yet. + !serializer; return options; } diff --git a/deps/v8/src/codegen/assembler.h b/deps/v8/src/codegen/assembler.h index 751799127d..7066905966 100644 --- a/deps/v8/src/codegen/assembler.h +++ b/deps/v8/src/codegen/assembler.h @@ -169,6 +169,11 @@ struct V8_EXPORT_PRIVATE AssemblerOptions { // Enables the use of isolate-independent builtins through an off-heap // trampoline. (macro assembler feature). bool inline_offheap_trampolines = true; + // Enables generation of pc-relative calls to builtins if the off-heap + // builtins are guaranteed to be within the reach of pc-relative call or jump + // instructions. For example, when the bultins code is re-embedded into the + // code range. + bool short_builtin_calls = false; // On some platforms, all code is within a given range in the process, // and the start of this range is configured here. Address code_range_start = 0; diff --git a/deps/v8/src/codegen/code-factory.cc b/deps/v8/src/codegen/code-factory.cc index ceabbac807..ece8200023 100644 --- a/deps/v8/src/codegen/code-factory.cc +++ b/deps/v8/src/codegen/code-factory.cc @@ -405,6 +405,13 @@ Callable CodeFactory::InterpreterOnStackReplacement(Isolate* isolate) { } // static +Callable CodeFactory::InterpreterOnStackReplacement_ToBaseline( + Isolate* isolate) { + return Builtins::CallableFor( + isolate, Builtins::kInterpreterOnStackReplacement_ToBaseline); +} + +// static Callable CodeFactory::ArrayNoArgumentConstructor( Isolate* isolate, ElementsKind kind, AllocationSiteOverrideMode override_mode) { diff --git a/deps/v8/src/codegen/code-factory.h b/deps/v8/src/codegen/code-factory.h index b98c576b9e..aab2977045 100644 --- a/deps/v8/src/codegen/code-factory.h +++ b/deps/v8/src/codegen/code-factory.h @@ -92,6 +92,7 @@ class V8_EXPORT_PRIVATE CodeFactory final { Isolate* isolate, InterpreterPushArgsMode mode); static Callable InterpreterCEntry(Isolate* isolate, int result_size = 1); static Callable InterpreterOnStackReplacement(Isolate* isolate); + static Callable InterpreterOnStackReplacement_ToBaseline(Isolate* isolate); static Callable ArrayNoArgumentConstructor( Isolate* isolate, ElementsKind kind, diff --git a/deps/v8/src/codegen/code-reference.cc b/deps/v8/src/codegen/code-reference.cc index 63c8d37497..0c550fa0d3 100644 --- a/deps/v8/src/codegen/code-reference.cc +++ b/deps/v8/src/codegen/code-reference.cc @@ -8,7 +8,10 @@ #include "src/common/globals.h" #include "src/handles/handles-inl.h" #include "src/objects/objects-inl.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY namespace v8 { namespace internal { @@ -28,6 +31,7 @@ struct JSOps { int code_comments_size() const { return code->code_comments_size(); } }; +#if V8_ENABLE_WEBASSEMBLY struct WasmOps { const wasm::WasmCode* code; @@ -48,6 +52,7 @@ struct WasmOps { Address code_comments() const { return code->code_comments(); } int code_comments_size() const { return code->code_comments_size(); } }; +#endif // V8_ENABLE_WEBASSEMBLY struct CodeDescOps { const CodeDesc* code_desc; @@ -76,6 +81,7 @@ struct CodeDescOps { }; } // namespace +#if V8_ENABLE_WEBASSEMBLY #define DISPATCH(ret, method) \ ret CodeReference::method() const { \ DCHECK(!is_null()); \ @@ -90,6 +96,18 @@ struct CodeDescOps { UNREACHABLE(); \ } \ } +#else +#define DISPATCH(ret, method) \ + ret CodeReference::method() const { \ + DCHECK(!is_null()); \ + DCHECK(kind_ == JS || kind_ == CODE_DESC); \ + if (kind_ == JS) { \ + return JSOps{js_code_}.method(); \ + } else { \ + return CodeDescOps{code_desc_}.method(); \ + } \ + } +#endif // V8_ENABLE_WEBASSEMBLY DISPATCH(Address, constant_pool) DISPATCH(Address, instruction_start) diff --git a/deps/v8/src/codegen/code-stub-assembler.cc b/deps/v8/src/codegen/code-stub-assembler.cc index 90e34d4dbd..0b039e40fa 100644 --- a/deps/v8/src/codegen/code-stub-assembler.cc +++ b/deps/v8/src/codegen/code-stub-assembler.cc @@ -4,6 +4,8 @@ #include "src/codegen/code-stub-assembler.h" +#include <functional> + #include "include/v8-internal.h" #include "src/base/macros.h" #include "src/codegen/code-factory.h" @@ -25,13 +27,14 @@ #include "src/objects/ordered-hash-table-inl.h" #include "src/objects/property-cell.h" #include "src/roots/roots.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-objects.h" +#endif // V8_ENABLE_WEBASSEMBLY namespace v8 { namespace internal { -using compiler::Node; - CodeStubAssembler::CodeStubAssembler(compiler::CodeAssemblerState* state) : compiler::CodeAssembler(state), TorqueGeneratedExportedMacrosAssembler(state) { @@ -349,7 +352,7 @@ TNode<Float64T> CodeStubAssembler::Float64Round(TNode<Float64T> x) { Goto(&return_x); BIND(&return_x); - return TNode<Float64T>::UncheckedCast(var_x.value()); + return var_x.value(); } TNode<Float64T> CodeStubAssembler::Float64Ceil(TNode<Float64T> x) { @@ -401,7 +404,7 @@ TNode<Float64T> CodeStubAssembler::Float64Ceil(TNode<Float64T> x) { Goto(&return_x); BIND(&return_x); - return TNode<Float64T>::UncheckedCast(var_x.value()); + return var_x.value(); } TNode<Float64T> CodeStubAssembler::Float64Floor(TNode<Float64T> x) { @@ -453,7 +456,7 @@ TNode<Float64T> CodeStubAssembler::Float64Floor(TNode<Float64T> x) { Goto(&return_x); BIND(&return_x); - return TNode<Float64T>::UncheckedCast(var_x.value()); + return var_x.value(); } TNode<Float64T> CodeStubAssembler::Float64RoundToEven(TNode<Float64T> x) { @@ -484,7 +487,7 @@ TNode<Float64T> CodeStubAssembler::Float64RoundToEven(TNode<Float64T> x) { Goto(&done); BIND(&done); - return TNode<Float64T>::UncheckedCast(var_result.value()); + return var_result.value(); } TNode<Float64T> CodeStubAssembler::Float64Trunc(TNode<Float64T> x) { @@ -545,7 +548,137 @@ TNode<Float64T> CodeStubAssembler::Float64Trunc(TNode<Float64T> x) { Goto(&return_x); BIND(&return_x); - return TNode<Float64T>::UncheckedCast(var_x.value()); + return var_x.value(); +} + +TNode<IntPtrT> CodeStubAssembler::PopulationCountFallback( + TNode<UintPtrT> value) { + // Taken from slow path of base::bits::CountPopulation, the comments here show + // C++ code and comments from there for reference. + // Fall back to divide-and-conquer popcount (see "Hacker's Delight" by Henry + // S. Warren, Jr.), chapter 5-1. + constexpr uintptr_t mask[] = {static_cast<uintptr_t>(0x5555555555555555), + static_cast<uintptr_t>(0x3333333333333333), + static_cast<uintptr_t>(0x0f0f0f0f0f0f0f0f)}; + + // TNode<UintPtrT> value = Unsigned(value_word); + TNode<UintPtrT> lhs, rhs; + + // Start with 64 buckets of 1 bits, holding values from [0,1]. + // {value = ((value >> 1) & mask[0]) + (value & mask[0])} + lhs = WordAnd(WordShr(value, UintPtrConstant(1)), UintPtrConstant(mask[0])); + rhs = WordAnd(value, UintPtrConstant(mask[0])); + value = UintPtrAdd(lhs, rhs); + + // Having 32 buckets of 2 bits, holding values from [0,2] now. + // {value = ((value >> 2) & mask[1]) + (value & mask[1])} + lhs = WordAnd(WordShr(value, UintPtrConstant(2)), UintPtrConstant(mask[1])); + rhs = WordAnd(value, UintPtrConstant(mask[1])); + value = UintPtrAdd(lhs, rhs); + + // Having 16 buckets of 4 bits, holding values from [0,4] now. + // {value = ((value >> 4) & mask[2]) + (value & mask[2])} + lhs = WordAnd(WordShr(value, UintPtrConstant(4)), UintPtrConstant(mask[2])); + rhs = WordAnd(value, UintPtrConstant(mask[2])); + value = UintPtrAdd(lhs, rhs); + + // Having 8 buckets of 8 bits, holding values from [0,8] now. + // From this point on, the buckets are bigger than the number of bits + // required to hold the values, and the buckets are bigger the maximum + // result, so there's no need to mask value anymore, since there's no + // more risk of overflow between buckets. + // {value = (value >> 8) + value} + lhs = WordShr(value, UintPtrConstant(8)); + value = UintPtrAdd(lhs, value); + + // Having 4 buckets of 16 bits, holding values from [0,16] now. + // {value = (value >> 16) + value} + lhs = WordShr(value, UintPtrConstant(16)); + value = UintPtrAdd(lhs, value); + + if (Is64()) { + // Having 2 buckets of 32 bits, holding values from [0,32] now. + // {value = (value >> 32) + value} + lhs = WordShr(value, UintPtrConstant(32)); + value = UintPtrAdd(lhs, value); + } + + // Having 1 buckets of sizeof(intptr_t) bits, holding values from [0,64] now. + // {return static_cast<unsigned>(value & 0xff)} + return Signed(WordAnd(value, UintPtrConstant(0xff))); +} + +TNode<Int64T> CodeStubAssembler::PopulationCount64(TNode<Word64T> value) { + if (IsWord64PopcntSupported()) { + return Word64Popcnt(value); + } + + if (Is32()) { + // Unsupported. + UNREACHABLE(); + } + + return ReinterpretCast<Int64T>( + PopulationCountFallback(ReinterpretCast<UintPtrT>(value))); +} + +TNode<Int32T> CodeStubAssembler::PopulationCount32(TNode<Word32T> value) { + if (IsWord32PopcntSupported()) { + return Word32Popcnt(value); + } + + if (Is32()) { + TNode<IntPtrT> res = + PopulationCountFallback(ReinterpretCast<UintPtrT>(value)); + return ReinterpretCast<Int32T>(res); + } else { + TNode<IntPtrT> res = PopulationCountFallback( + ReinterpretCast<UintPtrT>(ChangeUint32ToUint64(value))); + return TruncateInt64ToInt32(ReinterpretCast<Int64T>(res)); + } +} + +TNode<Int64T> CodeStubAssembler::CountTrailingZeros64(TNode<Word64T> value) { + if (IsWord64CtzSupported()) { + return Word64Ctz(value); + } + + if (Is32()) { + // Unsupported. + UNREACHABLE(); + } + + // Same fallback as in base::bits::CountTrailingZeros. + // Fall back to popcount (see "Hacker's Delight" by Henry S. Warren, Jr.), + // chapter 5-4. On x64, since is faster than counting in a loop and faster + // than doing binary search. + TNode<Word64T> lhs = Word64Not(value); + TNode<Word64T> rhs = Uint64Sub(Unsigned(value), Uint64Constant(1)); + return PopulationCount64(Word64And(lhs, rhs)); +} + +TNode<Int32T> CodeStubAssembler::CountTrailingZeros32(TNode<Word32T> value) { + if (IsWord32CtzSupported()) { + return Word32Ctz(value); + } + + if (Is32()) { + // Same fallback as in Word64CountTrailingZeros. + TNode<Word32T> lhs = Word32BitwiseNot(value); + TNode<Word32T> rhs = Int32Sub(Signed(value), Int32Constant(1)); + return PopulationCount32(Word32And(lhs, rhs)); + } else { + TNode<Int64T> res64 = CountTrailingZeros64(ChangeUint32ToUint64(value)); + return TruncateInt64ToInt32(Signed(res64)); + } +} + +TNode<Int64T> CodeStubAssembler::CountLeadingZeros64(TNode<Word64T> value) { + return Word64Clz(value); +} + +TNode<Int32T> CodeStubAssembler::CountLeadingZeros32(TNode<Word32T> value) { + return Word32Clz(value); } template <> @@ -610,7 +743,7 @@ TNode<Smi> CodeStubAssembler::NormalizeSmiIndex(TNode<Smi> smi_index) { return smi_index; } -TNode<Smi> CodeStubAssembler::SmiFromInt32(SloppyTNode<Int32T> value) { +TNode<Smi> CodeStubAssembler::SmiFromInt32(TNode<Int32T> value) { if (COMPRESS_POINTERS_BOOL) { static_assert(!COMPRESS_POINTERS_BOOL || (kSmiShiftSize + kSmiTagSize == 1), "Use shifting instead of add"); @@ -1590,16 +1723,16 @@ TNode<HeapObject> CodeStubAssembler::LoadSlowProperties( CSA_SLOW_ASSERT(this, IsDictionaryMap(LoadMap(object))); TNode<Object> properties = LoadJSReceiverPropertiesOrHash(object); NodeGenerator<HeapObject> make_empty = [=]() -> TNode<HeapObject> { - if (V8_DICT_MODE_PROTOTYPES_BOOL) { - return EmptyOrderedPropertyDictionaryConstant(); + if (V8_ENABLE_SWISS_NAME_DICTIONARY_BOOL) { + return EmptySwissPropertyDictionaryConstant(); } else { return EmptyPropertyDictionaryConstant(); } }; NodeGenerator<HeapObject> cast_properties = [=] { TNode<HeapObject> dict = CAST(properties); - if (V8_DICT_MODE_PROTOTYPES_BOOL) { - CSA_ASSERT(this, Word32Or(IsOrderedNameDictionary(dict), + if (V8_ENABLE_SWISS_NAME_DICTIONARY_BOOL) { + CSA_ASSERT(this, Word32Or(IsSwissNameDictionary(dict), IsGlobalDictionary(dict))); } else { CSA_ASSERT(this, @@ -1775,27 +1908,25 @@ TNode<Uint32T> CodeStubAssembler::EnsureOnlyHasSimpleProperties( } TNode<IntPtrT> CodeStubAssembler::LoadJSReceiverIdentityHash( - TNode<Object> receiver, Label* if_no_hash) { + TNode<JSReceiver> receiver, Label* if_no_hash) { TVARIABLE(IntPtrT, var_hash); Label done(this), if_smi(this), if_property_array(this), - if_ordered_property_dictionary(this), if_property_dictionary(this), + if_swiss_property_dictionary(this), if_property_dictionary(this), if_fixed_array(this); TNode<Object> properties_or_hash = - LoadObjectField(TNode<HeapObject>::UncheckedCast(receiver), - JSReceiver::kPropertiesOrHashOffset); + LoadObjectField(receiver, JSReceiver::kPropertiesOrHashOffset); GotoIf(TaggedIsSmi(properties_or_hash), &if_smi); - TNode<HeapObject> properties = - TNode<HeapObject>::UncheckedCast(properties_or_hash); + TNode<HeapObject> properties = CAST(properties_or_hash); TNode<Uint16T> properties_instance_type = LoadInstanceType(properties); GotoIf(InstanceTypeEqual(properties_instance_type, PROPERTY_ARRAY_TYPE), &if_property_array); - if (V8_DICT_MODE_PROTOTYPES_BOOL) { - GotoIf(InstanceTypeEqual(properties_instance_type, - ORDERED_NAME_DICTIONARY_TYPE), - &if_ordered_property_dictionary); + if (V8_ENABLE_SWISS_NAME_DICTIONARY_BOOL) { + GotoIf( + InstanceTypeEqual(properties_instance_type, SWISS_NAME_DICTIONARY_TYPE), + &if_swiss_property_dictionary); } Branch(InstanceTypeEqual(properties_instance_type, NAME_DICTIONARY_TYPE), &if_property_dictionary, &if_fixed_array); @@ -1808,7 +1939,7 @@ TNode<IntPtrT> CodeStubAssembler::LoadJSReceiverIdentityHash( BIND(&if_smi); { - var_hash = SmiUntag(TNode<Smi>::UncheckedCast(properties_or_hash)); + var_hash = SmiUntag(CAST(properties_or_hash)); Goto(&done); } @@ -1816,15 +1947,14 @@ TNode<IntPtrT> CodeStubAssembler::LoadJSReceiverIdentityHash( { TNode<IntPtrT> length_and_hash = LoadAndUntagObjectField( properties, PropertyArray::kLengthAndHashOffset); - var_hash = TNode<IntPtrT>::UncheckedCast( - DecodeWord<PropertyArray::HashField>(length_and_hash)); + var_hash = Signed(DecodeWord<PropertyArray::HashField>(length_and_hash)); Goto(&done); } - if (V8_DICT_MODE_PROTOTYPES_BOOL) { - BIND(&if_ordered_property_dictionary); + if (V8_ENABLE_SWISS_NAME_DICTIONARY_BOOL) { + BIND(&if_swiss_property_dictionary); { - var_hash = SmiUntag(CAST(LoadFixedArrayElement( - CAST(properties), OrderedNameDictionary::HashIndex()))); + var_hash = Signed( + ChangeUint32ToWord(LoadSwissNameDictionaryHash(CAST(properties)))); Goto(&done); } } @@ -2694,9 +2824,8 @@ TNode<BoolT> CodeStubAssembler::IsGeneratorFunction( TNode<BoolT> CodeStubAssembler::IsJSFunctionWithPrototypeSlot( TNode<HeapObject> object) { // Only JSFunction maps may have HasPrototypeSlotBit set. - return TNode<BoolT>::UncheckedCast( - IsSetWord32<Map::Bits1::HasPrototypeSlotBit>( - LoadMapBitField(LoadMap(object)))); + return IsSetWord32<Map::Bits1::HasPrototypeSlotBit>( + LoadMapBitField(LoadMap(object))); } void CodeStubAssembler::BranchIfHasPrototypeProperty( @@ -2784,9 +2913,19 @@ void CodeStubAssembler::StoreHeapNumberValue(TNode<HeapNumber> object, } void CodeStubAssembler::StoreObjectField(TNode<HeapObject> object, int offset, + TNode<Smi> value) { + StoreObjectFieldNoWriteBarrier(object, offset, value); +} + +void CodeStubAssembler::StoreObjectField(TNode<HeapObject> object, + TNode<IntPtrT> offset, + TNode<Smi> value) { + StoreObjectFieldNoWriteBarrier(object, offset, value); +} + +void CodeStubAssembler::StoreObjectField(TNode<HeapObject> object, int offset, TNode<Object> value) { DCHECK_NE(HeapObject::kMapOffset, offset); // Use StoreMap instead. - OptimizedStoreField(MachineRepresentation::kTagged, UncheckedCast<HeapObject>(object), offset, value); } @@ -2867,7 +3006,7 @@ void CodeStubAssembler::StoreFixedArrayOrPropertyArrayElement( [=] { TNode<IntPtrT> length_and_hash = LoadAndUntagObjectField( object, PropertyArray::kLengthAndHashOffset); - return TNode<IntPtrT>::UncheckedCast( + return Signed( DecodeWord<PropertyArray::LengthField>(length_and_hash)); }, [=] { @@ -3109,7 +3248,7 @@ TNode<HeapNumber> CodeStubAssembler::AllocateHeapNumber() { } TNode<HeapNumber> CodeStubAssembler::AllocateHeapNumberWithValue( - SloppyTNode<Float64T> value) { + TNode<Float64T> value) { TNode<HeapNumber> result = AllocateHeapNumber(); StoreHeapNumberValue(result, value); return result; @@ -3205,8 +3344,33 @@ TNode<UintPtrT> CodeStubAssembler::LoadBigIntDigit(TNode<BigInt> bigint, return LoadObjectField<UintPtrT>(bigint, offset); } +TNode<ByteArray> CodeStubAssembler::AllocateNonEmptyByteArray( + TNode<UintPtrT> length, AllocationFlags flags) { + CSA_ASSERT(this, WordNotEqual(length, IntPtrConstant(0))); + + Comment("AllocateNonEmptyByteArray"); + TVARIABLE(Object, var_result); + + TNode<IntPtrT> raw_size = + GetArrayAllocationSize(Signed(length), UINT8_ELEMENTS, + ByteArray::kHeaderSize + kObjectAlignmentMask); + TNode<IntPtrT> size = + WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask)); + + TNode<HeapObject> result = Allocate(size, flags); + + DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kByteArrayMap)); + StoreMapNoWriteBarrier(result, RootIndex::kByteArrayMap); + StoreObjectFieldNoWriteBarrier(result, ByteArray::kLengthOffset, + SmiTag(Signed(length))); + + return CAST(result); +} + TNode<ByteArray> CodeStubAssembler::AllocateByteArray(TNode<UintPtrT> length, AllocationFlags flags) { + // TODO(ishell): unify with AllocateNonEmptyByteArray(). + Comment("AllocateByteArray"); TVARIABLE(Object, var_result); @@ -3584,7 +3748,7 @@ void CodeStubAssembler::InitializeJSObjectFromMap( } else { CSA_ASSERT(this, Word32Or(Word32Or(Word32Or(IsPropertyArray(*properties), IsNameDictionary(*properties)), - IsOrderedNameDictionary(*properties)), + IsSwissNameDictionary(*properties)), IsEmptyFixedArray(*properties))); StoreObjectFieldNoWriteBarrier(object, JSObject::kPropertiesOrHashOffset, *properties); @@ -5423,8 +5587,7 @@ TNode<Number> CodeStubAssembler::ChangeFloat64ToTagged(TNode<Float64T> value) { return var_result.value(); } -TNode<Number> CodeStubAssembler::ChangeInt32ToTagged( - SloppyTNode<Int32T> value) { +TNode<Number> CodeStubAssembler::ChangeInt32ToTagged(TNode<Int32T> value) { if (SmiValuesAre32Bits()) { return SmiTag(ChangeInt32ToIntPtr(value)); } @@ -5454,8 +5617,7 @@ TNode<Number> CodeStubAssembler::ChangeInt32ToTagged( return var_result.value(); } -TNode<Number> CodeStubAssembler::ChangeUint32ToTagged( - SloppyTNode<Uint32T> value) { +TNode<Number> CodeStubAssembler::ChangeUint32ToTagged(TNode<Uint32T> value) { Label if_overflow(this, Label::kDeferred), if_not_overflow(this), if_join(this); TVARIABLE(Number, var_result); @@ -5507,6 +5669,10 @@ TNode<Number> CodeStubAssembler::ChangeUintPtrToTagged(TNode<UintPtrT> value) { return var_result.value(); } +TNode<Int32T> CodeStubAssembler::ChangeBoolToInt32(TNode<BoolT> b) { + return UncheckedCast<Int32T>(b); +} + TNode<String> CodeStubAssembler::ToThisString(TNode<Context> context, TNode<Object> value, TNode<String> method_name) { @@ -6402,6 +6568,11 @@ TNode<BoolT> CodeStubAssembler::IsOrderedNameDictionary( return HasInstanceType(object, ORDERED_NAME_DICTIONARY_TYPE); } +TNode<BoolT> CodeStubAssembler::IsSwissNameDictionary( + TNode<HeapObject> object) { + return HasInstanceType(object, SWISS_NAME_DICTIONARY_TYPE); +} + TNode<BoolT> CodeStubAssembler::IsGlobalDictionary(TNode<HeapObject> object) { return HasInstanceType(object, GLOBAL_DICTIONARY_TYPE); } @@ -7858,6 +8029,108 @@ TNode<MaybeObject> CodeStubAssembler::LoadFieldTypeByDescriptorEntry( DescriptorArray::ToValueIndex(0) * kTaggedSize); } +// Loads the value for the entry with the given key_index. +// Returns a tagged value. +template <class ContainerType> +TNode<Object> CodeStubAssembler::LoadValueByKeyIndex( + TNode<ContainerType> container, TNode<IntPtrT> key_index) { + static_assert(!std::is_same<ContainerType, DescriptorArray>::value, + "Use the non-templatized version for DescriptorArray"); + const int kKeyToValueOffset = + (ContainerType::kEntryValueIndex - ContainerType::kEntryKeyIndex) * + kTaggedSize; + return LoadFixedArrayElement(container, key_index, kKeyToValueOffset); +} + +template <> +V8_EXPORT_PRIVATE TNode<Object> CodeStubAssembler::LoadValueByKeyIndex( + TNode<SwissNameDictionary> container, TNode<IntPtrT> key_index) { + TNode<IntPtrT> offset_minus_tag = SwissNameDictionaryOffsetIntoDataTableMT( + container, key_index, SwissNameDictionary::kDataTableValueEntryIndex); + + return Load<Object>(container, offset_minus_tag); +} + +template <class ContainerType> +TNode<Uint32T> CodeStubAssembler::LoadDetailsByKeyIndex( + TNode<ContainerType> container, TNode<IntPtrT> key_index) { + static_assert(!std::is_same<ContainerType, DescriptorArray>::value, + "Use the non-templatized version for DescriptorArray"); + const int kKeyToDetailsOffset = + (ContainerType::kEntryDetailsIndex - ContainerType::kEntryKeyIndex) * + kTaggedSize; + return Unsigned(LoadAndUntagToWord32FixedArrayElement(container, key_index, + kKeyToDetailsOffset)); +} + +template <> +V8_EXPORT_PRIVATE TNode<Uint32T> CodeStubAssembler::LoadDetailsByKeyIndex( + TNode<SwissNameDictionary> container, TNode<IntPtrT> key_index) { + TNode<IntPtrT> capacity = + ChangeInt32ToIntPtr(LoadSwissNameDictionaryCapacity(container)); + return LoadSwissNameDictionaryPropertyDetails(container, capacity, key_index); +} + +// Stores the details for the entry with the given key_index. +// |details| must be a Smi. +template <class ContainerType> +void CodeStubAssembler::StoreDetailsByKeyIndex(TNode<ContainerType> container, + TNode<IntPtrT> key_index, + TNode<Smi> details) { + const int kKeyToDetailsOffset = + (ContainerType::kEntryDetailsIndex - ContainerType::kEntryKeyIndex) * + kTaggedSize; + StoreFixedArrayElement(container, key_index, details, kKeyToDetailsOffset); +} + +template <> +V8_EXPORT_PRIVATE void CodeStubAssembler::StoreDetailsByKeyIndex( + TNode<SwissNameDictionary> container, TNode<IntPtrT> key_index, + TNode<Smi> details) { + TNode<IntPtrT> capacity = + ChangeInt32ToIntPtr(LoadSwissNameDictionaryCapacity(container)); + TNode<Uint8T> details_byte = UncheckedCast<Uint8T>(SmiToInt32(details)); + StoreSwissNameDictionaryPropertyDetails(container, capacity, key_index, + details_byte); +} + +// Stores the value for the entry with the given key_index. +template <class ContainerType> +void CodeStubAssembler::StoreValueByKeyIndex(TNode<ContainerType> container, + TNode<IntPtrT> key_index, + TNode<Object> value, + WriteBarrierMode write_barrier) { + const int kKeyToValueOffset = + (ContainerType::kEntryValueIndex - ContainerType::kEntryKeyIndex) * + kTaggedSize; + StoreFixedArrayElement(container, key_index, value, write_barrier, + kKeyToValueOffset); +} + +template <> +V8_EXPORT_PRIVATE void CodeStubAssembler::StoreValueByKeyIndex( + TNode<SwissNameDictionary> container, TNode<IntPtrT> key_index, + TNode<Object> value, WriteBarrierMode write_barrier) { + TNode<IntPtrT> offset_minus_tag = SwissNameDictionaryOffsetIntoDataTableMT( + container, key_index, SwissNameDictionary::kDataTableValueEntryIndex); + + StoreToObjectWriteBarrier mode; + switch (write_barrier) { + case UNSAFE_SKIP_WRITE_BARRIER: + case SKIP_WRITE_BARRIER: + mode = StoreToObjectWriteBarrier::kNone; + break; + case UPDATE_WRITE_BARRIER: + mode = StoreToObjectWriteBarrier::kFull; + break; + default: + // We shouldn't see anything else. + UNREACHABLE(); + } + StoreToObject(MachineRepresentation::kTagged, container, offset_minus_tag, + value, mode); +} + template V8_EXPORT_PRIVATE TNode<IntPtrT> CodeStubAssembler::EntryToIndex<NameDictionary>(TNode<IntPtrT>, int); template V8_EXPORT_PRIVATE TNode<IntPtrT> @@ -7865,6 +8138,19 @@ CodeStubAssembler::EntryToIndex<GlobalDictionary>(TNode<IntPtrT>, int); template V8_EXPORT_PRIVATE TNode<IntPtrT> CodeStubAssembler::EntryToIndex<NumberDictionary>(TNode<IntPtrT>, int); +template TNode<Object> CodeStubAssembler::LoadValueByKeyIndex( + TNode<NameDictionary> container, TNode<IntPtrT> key_index); +template TNode<Object> CodeStubAssembler::LoadValueByKeyIndex( + TNode<GlobalDictionary> container, TNode<IntPtrT> key_index); +template TNode<Uint32T> CodeStubAssembler::LoadDetailsByKeyIndex( + TNode<NameDictionary> container, TNode<IntPtrT> key_index); +template void CodeStubAssembler::StoreDetailsByKeyIndex( + TNode<NameDictionary> container, TNode<IntPtrT> key_index, + TNode<Smi> details); +template void CodeStubAssembler::StoreValueByKeyIndex( + TNode<NameDictionary> container, TNode<IntPtrT> key_index, + TNode<Object> value, WriteBarrierMode write_barrier); + // This must be kept in sync with HashTableBase::ComputeCapacity(). TNode<IntPtrT> CodeStubAssembler::HashTableComputeCapacity( TNode<IntPtrT> at_least_space_for) { @@ -8010,6 +8296,15 @@ TNode<Word32T> CodeStubAssembler::ComputeSeededHash(TNode<IntPtrT> key) { std::make_pair(type_int32, TruncateIntPtrToInt32(key)))); } +template <> +void CodeStubAssembler::NameDictionaryLookup( + TNode<SwissNameDictionary> dictionary, TNode<Name> unique_name, + Label* if_found, TVariable<IntPtrT>* var_name_index, Label* if_not_found, + LookupMode mode) { + SwissNameDictionaryFindEntry(dictionary, unique_name, if_found, + var_name_index, if_not_found); +} + void CodeStubAssembler::NumberDictionaryLookup( TNode<NumberDictionary> dictionary, TNode<IntPtrT> intptr_index, Label* if_found, TVariable<IntPtrT>* var_entry, Label* if_not_found) { @@ -8196,6 +8491,31 @@ void CodeStubAssembler::Add(TNode<Dictionary> dictionary, TNode<Name> key, enum_index); } +template <> +void CodeStubAssembler::Add(TNode<SwissNameDictionary> dictionary, + TNode<Name> key, TNode<Object> value, + Label* bailout) { + PropertyDetails d(kData, NONE, + PropertyDetails::kConstIfDictConstnessTracking); + + PropertyDetails d_dont_enum(kData, DONT_ENUM, + PropertyDetails::kConstIfDictConstnessTracking); + TNode<Uint8T> details_byte_enum = + UncheckedCast<Uint8T>(Uint32Constant(d.ToByte())); + TNode<Uint8T> details_byte_dont_enum = + UncheckedCast<Uint8T>(Uint32Constant(d_dont_enum.ToByte())); + + Label not_private(this); + TVARIABLE(Uint8T, var_details, details_byte_enum); + + GotoIfNot(IsPrivateSymbol(key), ¬_private); + var_details = details_byte_dont_enum; + Goto(¬_private); + + BIND(¬_private); + SwissNameDictionaryAdd(dictionary, key, value, var_details.value(), bailout); +} + template void CodeStubAssembler::Add<NameDictionary>(TNode<NameDictionary>, TNode<Name>, TNode<Object>, Label*); @@ -8209,9 +8529,11 @@ TNode<Smi> CodeStubAssembler::GetNumberOfElements( template <> TNode<Smi> CodeStubAssembler::GetNumberOfElements( - TNode<OrderedNameDictionary> dictionary) { - return CAST(LoadFixedArrayElement( - dictionary, OrderedNameDictionary::NumberOfElementsIndex())); + TNode<SwissNameDictionary> dictionary) { + TNode<IntPtrT> capacity = + ChangeInt32ToIntPtr(LoadSwissNameDictionaryCapacity(dictionary)); + return SmiFromIntPtr( + LoadSwissNameDictionaryNumberOfElements(dictionary, capacity)); } template TNode<Smi> CodeStubAssembler::GetNumberOfElements( @@ -8528,13 +8850,8 @@ void CodeStubAssembler::ForEachEnumerableOwnProperty( } BIND(&if_found_dict); { - if (V8_DICT_MODE_PROTOTYPES_BOOL) { - // TODO(v8:11167, v8:11177) Only here due to SetDataProperties - // workaround. - GotoIf(Int32TrueConstant(), bailout); - } - - TNode<NameDictionary> dictionary = CAST(var_meta_storage.value()); + TNode<PropertyDictionary> dictionary = + CAST(var_meta_storage.value()); TNode<IntPtrT> entry = var_entry.value(); TNode<Uint32T> details = LoadDetailsByKeyIndex(dictionary, entry); @@ -8544,7 +8861,8 @@ void CodeStubAssembler::ForEachEnumerableOwnProperty( &next_iteration); var_details = details; - var_value = LoadValueByKeyIndex<NameDictionary>(dictionary, entry); + var_value = + LoadValueByKeyIndex<PropertyDictionary>(dictionary, entry); Goto(&if_found); } @@ -8733,16 +9051,11 @@ void CodeStubAssembler::TryLookupPropertyInSimpleObject( } BIND(&if_isslowmap); { - if (V8_DICT_MODE_PROTOTYPES_BOOL) { - // TODO(v8:11167, v8:11177) Only here due to SetDataProperties workaround. - GotoIf(Int32TrueConstant(), bailout); - } - - TNode<NameDictionary> dictionary = CAST(LoadSlowProperties(object)); + TNode<PropertyDictionary> dictionary = CAST(LoadSlowProperties(object)); *var_meta_storage = dictionary; - NameDictionaryLookup<NameDictionary>(dictionary, unique_name, if_found_dict, - var_name_index, if_not_found); + NameDictionaryLookup<PropertyDictionary>( + dictionary, unique_name, if_found_dict, var_name_index, if_not_found); } } @@ -8966,8 +9279,9 @@ void CodeStubAssembler::LoadPropertyFromFastObject( Comment("] LoadPropertyFromFastObject"); } -void CodeStubAssembler::LoadPropertyFromNameDictionary( - TNode<NameDictionary> dictionary, TNode<IntPtrT> name_index, +template <typename Dictionary> +void CodeStubAssembler::LoadPropertyFromDictionary( + TNode<Dictionary> dictionary, TNode<IntPtrT> name_index, TVariable<Uint32T>* var_details, TVariable<Object>* var_value) { Comment("LoadPropertyFromNameDictionary"); *var_details = LoadDetailsByKeyIndex(dictionary, name_index); @@ -8997,6 +9311,14 @@ void CodeStubAssembler::LoadPropertyFromGlobalDictionary( Comment("] LoadPropertyFromGlobalDictionary"); } +template void CodeStubAssembler::LoadPropertyFromDictionary( + TNode<NameDictionary> dictionary, TNode<IntPtrT> name_index, + TVariable<Uint32T>* var_details, TVariable<Object>* var_value); + +template void CodeStubAssembler::LoadPropertyFromDictionary( + TNode<SwissNameDictionary> dictionary, TNode<IntPtrT> name_index, + TVariable<Uint32T>* var_details, TVariable<Object>* var_value); + // |value| is the property backing store's contents, which is either a value or // an accessor pair, as specified by |details|. |holder| is a JSObject or a // PropertyCell (TODO: use UnionT). Returns either the original value, or the @@ -9159,9 +9481,10 @@ void CodeStubAssembler::TryGetOwnProperty( } BIND(&if_found_dict); { - TNode<NameDictionary> dictionary = CAST(var_meta_storage.value()); + TNode<PropertyDictionary> dictionary = CAST(var_meta_storage.value()); TNode<IntPtrT> entry = var_entry.value(); - LoadPropertyFromNameDictionary(dictionary, entry, var_details, var_value); + LoadPropertyFromDictionary(dictionary, entry, var_details, var_value); + Goto(&if_found); } BIND(&if_found_global); @@ -9284,7 +9607,7 @@ void CodeStubAssembler::TryLookupElement( { // Negative and too-large keys must be converted to property names. if (Is64()) { - GotoIf(UintPtrLessThan(IntPtrConstant(JSArray::kMaxArrayIndex), + GotoIf(UintPtrLessThan(IntPtrConstant(JSObject::kMaxElementIndex), intptr_index), if_bailout); } else { @@ -9323,7 +9646,7 @@ void CodeStubAssembler::TryLookupElement( // Positive OOB indices mean "not found", negative indices and indices // out of array index range must be converted to property names. if (Is64()) { - GotoIf(UintPtrLessThan(IntPtrConstant(JSArray::kMaxArrayIndex), + GotoIf(UintPtrLessThan(IntPtrConstant(JSObject::kMaxElementIndex), intptr_index), if_bailout); } else { @@ -9803,7 +10126,7 @@ void CodeStubAssembler::UpdateFeedback(TNode<Smi> feedback, } void CodeStubAssembler::ReportFeedbackUpdate( - TNode<FeedbackVector> feedback_vector, SloppyTNode<UintPtrT> slot_id, + TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot_id, const char* reason) { // Reset profiler ticks. StoreObjectFieldNoWriteBarrier( @@ -9940,28 +10263,25 @@ MachineRepresentation ElementsKindToMachineRepresentation(ElementsKind kind) { } // namespace -template <typename TArray, typename TIndex> -void CodeStubAssembler::StoreElementTypedArray(TNode<TArray> elements, - ElementsKind kind, - TNode<TIndex> index, - Node* value) { - // TODO(v8:9708): Do we want to keep both IntPtrT and UintPtrT variants? - static_assert(std::is_same<TIndex, Smi>::value || - std::is_same<TIndex, UintPtrT>::value || +// TODO(solanes): Since we can't use `if constexpr` until we enable C++17 we +// have to specialize the BigInt and Word32T cases. Since we can't partly +// specialize, we have to specialize all used combinations. +template <typename TIndex> +void CodeStubAssembler::StoreElementTypedArrayBigInt(TNode<RawPtrT> elements, + ElementsKind kind, + TNode<TIndex> index, + TNode<BigInt> value) { + static_assert(std::is_same<TIndex, UintPtrT>::value || std::is_same<TIndex, IntPtrT>::value, - "Only Smi, UintPtrT or IntPtrT index is allowed"); - static_assert(std::is_same<TArray, RawPtrT>::value || - std::is_same<TArray, FixedArrayBase>::value, - "Only RawPtrT or FixedArrayBase elements are allowed"); - DCHECK(IsTypedArrayElementsKind(kind)); - if (kind == BIGINT64_ELEMENTS || kind == BIGUINT64_ELEMENTS) { - TNode<IntPtrT> offset = ElementOffsetFromIndex(index, kind, 0); - TVARIABLE(UintPtrT, var_low); - // Only used on 32-bit platforms. - TVARIABLE(UintPtrT, var_high); - BigIntToRawBytes(CAST(value), &var_low, &var_high); - - MachineRepresentation rep = WordT::kMachineRepresentation; + "Only UintPtrT or IntPtrT indices is allowed"); + DCHECK(kind == BIGINT64_ELEMENTS || kind == BIGUINT64_ELEMENTS); + TNode<IntPtrT> offset = ElementOffsetFromIndex(index, kind, 0); + TVARIABLE(UintPtrT, var_low); + // Only used on 32-bit platforms. + TVARIABLE(UintPtrT, var_high); + BigIntToRawBytes(value, &var_low, &var_high); + + MachineRepresentation rep = WordT::kMachineRepresentation; #if defined(V8_TARGET_BIG_ENDIAN) if (!Is64()) { StoreNoWriteBarrier(rep, elements, offset, var_high.value()); @@ -9979,16 +10299,82 @@ void CodeStubAssembler::StoreElementTypedArray(TNode<TArray> elements, var_high.value()); } #endif - } else { - if (kind == UINT8_CLAMPED_ELEMENTS) { - CSA_ASSERT(this, Word32Equal(UncheckedCast<Word32T>(value), - Word32And(Int32Constant(0xFF), value))); - } - TNode<IntPtrT> offset = ElementOffsetFromIndex(index, kind, 0); - // TODO(cbruni): Add OOB check once typed. - MachineRepresentation rep = ElementsKindToMachineRepresentation(kind); - StoreNoWriteBarrier(rep, elements, offset, value); +} + +template <> +void CodeStubAssembler::StoreElementTypedArray(TNode<RawPtrT> elements, + ElementsKind kind, + TNode<UintPtrT> index, + TNode<BigInt> value) { + StoreElementTypedArrayBigInt(elements, kind, index, value); +} + +template <> +void CodeStubAssembler::StoreElementTypedArray(TNode<RawPtrT> elements, + ElementsKind kind, + TNode<IntPtrT> index, + TNode<BigInt> value) { + StoreElementTypedArrayBigInt(elements, kind, index, value); +} + +template <typename TIndex> +void CodeStubAssembler::StoreElementTypedArrayWord32(TNode<RawPtrT> elements, + ElementsKind kind, + TNode<TIndex> index, + TNode<Word32T> value) { + static_assert(std::is_same<TIndex, UintPtrT>::value || + std::is_same<TIndex, IntPtrT>::value, + "Only UintPtrT or IntPtrT indices is allowed"); + DCHECK(IsTypedArrayElementsKind(kind)); + if (kind == UINT8_CLAMPED_ELEMENTS) { + CSA_ASSERT(this, Word32Equal(value, Word32And(Int32Constant(0xFF), value))); } + TNode<IntPtrT> offset = ElementOffsetFromIndex(index, kind, 0); + // TODO(cbruni): Add OOB check once typed. + MachineRepresentation rep = ElementsKindToMachineRepresentation(kind); + StoreNoWriteBarrier(rep, elements, offset, value); +} + +template <> +void CodeStubAssembler::StoreElementTypedArray(TNode<RawPtrT> elements, + ElementsKind kind, + TNode<UintPtrT> index, + TNode<Word32T> value) { + StoreElementTypedArrayWord32(elements, kind, index, value); +} + +template <> +void CodeStubAssembler::StoreElementTypedArray(TNode<RawPtrT> elements, + ElementsKind kind, + TNode<IntPtrT> index, + TNode<Word32T> value) { + StoreElementTypedArrayWord32(elements, kind, index, value); +} + +template <typename TArray, typename TIndex, typename TValue> +void CodeStubAssembler::StoreElementTypedArray(TNode<TArray> elements, + ElementsKind kind, + TNode<TIndex> index, + TNode<TValue> value) { + // TODO(v8:9708): Do we want to keep both IntPtrT and UintPtrT variants? + static_assert(std::is_same<TIndex, Smi>::value || + std::is_same<TIndex, UintPtrT>::value || + std::is_same<TIndex, IntPtrT>::value, + "Only Smi, UintPtrT or IntPtrT indices is allowed"); + static_assert(std::is_same<TArray, RawPtrT>::value || + std::is_same<TArray, FixedArrayBase>::value, + "Only RawPtrT or FixedArrayBase elements are allowed"); + static_assert(std::is_same<TValue, Int32T>::value || + std::is_same<TValue, Float32T>::value || + std::is_same<TValue, Float64T>::value || + std::is_same<TValue, Object>::value, + "Only Int32T, Float32T, Float64T or object value " + "types are allowed"); + DCHECK(IsTypedArrayElementsKind(kind)); + TNode<IntPtrT> offset = ElementOffsetFromIndex(index, kind, 0); + // TODO(cbruni): Add OOB check once typed. + MachineRepresentation rep = ElementsKindToMachineRepresentation(kind); + StoreNoWriteBarrier(rep, elements, offset, value); } template <typename TIndex> @@ -10020,14 +10406,41 @@ void CodeStubAssembler::StoreElement(TNode<FixedArrayBase> elements, StoreFixedDoubleArrayElement(CAST(elements), index, value); } -template <typename TIndex> +template <typename TIndex, typename TValue> void CodeStubAssembler::StoreElement(TNode<RawPtrT> elements, ElementsKind kind, - TNode<TIndex> index, Node* value) { + TNode<TIndex> index, TNode<TValue> value) { + static_assert(std::is_same<TIndex, Smi>::value || + std::is_same<TIndex, IntPtrT>::value || + std::is_same<TIndex, UintPtrT>::value, + "Only Smi, IntPtrT or UintPtrT indices are allowed"); + static_assert( + std::is_same<TValue, Int32T>::value || + std::is_same<TValue, Word32T>::value || + std::is_same<TValue, Float32T>::value || + std::is_same<TValue, Float64T>::value || + std::is_same<TValue, BigInt>::value, + "Only Int32T, Word32T, Float32T, Float64T or BigInt value types " + "are allowed"); + DCHECK(IsTypedArrayElementsKind(kind)); StoreElementTypedArray(elements, kind, index, value); } -template V8_EXPORT_PRIVATE void CodeStubAssembler::StoreElement<UintPtrT>( - TNode<RawPtrT>, ElementsKind, TNode<UintPtrT>, Node*); +template V8_EXPORT_PRIVATE void CodeStubAssembler::StoreElement(TNode<RawPtrT>, + ElementsKind, + TNode<UintPtrT>, + TNode<Int32T>); +template V8_EXPORT_PRIVATE void CodeStubAssembler::StoreElement(TNode<RawPtrT>, + ElementsKind, + TNode<UintPtrT>, + TNode<Word32T>); +template V8_EXPORT_PRIVATE void CodeStubAssembler::StoreElement( + TNode<RawPtrT>, ElementsKind, TNode<UintPtrT>, TNode<Float32T>); +template V8_EXPORT_PRIVATE void CodeStubAssembler::StoreElement( + TNode<RawPtrT>, ElementsKind, TNode<UintPtrT>, TNode<Float64T>); +template V8_EXPORT_PRIVATE void CodeStubAssembler::StoreElement(TNode<RawPtrT>, + ElementsKind, + TNode<UintPtrT>, + TNode<BigInt>); TNode<Uint8T> CodeStubAssembler::Int32ToUint8Clamped( TNode<Int32T> int32_value) { @@ -10231,60 +10644,6 @@ TNode<BigInt> CodeStubAssembler::PrepareValueForWriteToTypedArray<BigInt>( return ToBigInt(context, input); } -template <> -TNode<UntaggedT> CodeStubAssembler::PrepareValueForWriteToTypedArray( - TNode<Object> input, ElementsKind elements_kind, TNode<Context> context) { - DCHECK(IsTypedArrayElementsKind(elements_kind)); - - switch (elements_kind) { - case UINT8_ELEMENTS: - case INT8_ELEMENTS: - case UINT16_ELEMENTS: - case INT16_ELEMENTS: - case UINT32_ELEMENTS: - case INT32_ELEMENTS: - case UINT8_CLAMPED_ELEMENTS: - return PrepareValueForWriteToTypedArray<Word32T>(input, elements_kind, - context); - case FLOAT32_ELEMENTS: - return PrepareValueForWriteToTypedArray<Float32T>(input, elements_kind, - context); - case FLOAT64_ELEMENTS: - return PrepareValueForWriteToTypedArray<Float64T>(input, elements_kind, - context); - default: - UNREACHABLE(); - } -} - -Node* CodeStubAssembler::PrepareValueForWriteToTypedArray( - TNode<Object> input, ElementsKind elements_kind, TNode<Context> context) { - DCHECK(IsTypedArrayElementsKind(elements_kind)); - - switch (elements_kind) { - case UINT8_ELEMENTS: - case INT8_ELEMENTS: - case UINT16_ELEMENTS: - case INT16_ELEMENTS: - case UINT32_ELEMENTS: - case INT32_ELEMENTS: - case UINT8_CLAMPED_ELEMENTS: - return PrepareValueForWriteToTypedArray<Word32T>(input, elements_kind, - context); - case FLOAT32_ELEMENTS: - return PrepareValueForWriteToTypedArray<Float32T>(input, elements_kind, - context); - case FLOAT64_ELEMENTS: - return PrepareValueForWriteToTypedArray<Float64T>(input, elements_kind, - context); - case BIGINT64_ELEMENTS: - case BIGUINT64_ELEMENTS: - return ToBigInt(context, input); - default: - UNREACHABLE(); - } -} - void CodeStubAssembler::BigIntToRawBytes(TNode<BigInt> bigint, TVariable<UintPtrT>* var_low, TVariable<UintPtrT>* var_high) { @@ -10318,6 +10677,134 @@ void CodeStubAssembler::BigIntToRawBytes(TNode<BigInt> bigint, BIND(&done); } +template <> +void CodeStubAssembler::EmitElementStoreTypedArrayUpdateValue( + TNode<Object> value, ElementsKind elements_kind, + TNode<Word32T> converted_value, TVariable<Object>* maybe_converted_value) { + switch (elements_kind) { + case UINT8_ELEMENTS: + case INT8_ELEMENTS: + case UINT16_ELEMENTS: + case INT16_ELEMENTS: + case UINT8_CLAMPED_ELEMENTS: + *maybe_converted_value = + SmiFromInt32(UncheckedCast<Int32T>(converted_value)); + break; + case UINT32_ELEMENTS: + *maybe_converted_value = + ChangeUint32ToTagged(UncheckedCast<Uint32T>(converted_value)); + break; + case INT32_ELEMENTS: + *maybe_converted_value = + ChangeInt32ToTagged(UncheckedCast<Int32T>(converted_value)); + break; + default: + UNREACHABLE(); + } +} + +template <> +void CodeStubAssembler::EmitElementStoreTypedArrayUpdateValue( + TNode<Object> value, ElementsKind elements_kind, + TNode<Float32T> converted_value, TVariable<Object>* maybe_converted_value) { + Label dont_allocate_heap_number(this), end(this); + GotoIf(TaggedIsSmi(value), &dont_allocate_heap_number); + GotoIf(IsHeapNumber(CAST(value)), &dont_allocate_heap_number); + { + *maybe_converted_value = + AllocateHeapNumberWithValue(ChangeFloat32ToFloat64(converted_value)); + Goto(&end); + } + BIND(&dont_allocate_heap_number); + { + *maybe_converted_value = value; + Goto(&end); + } + BIND(&end); +} + +template <> +void CodeStubAssembler::EmitElementStoreTypedArrayUpdateValue( + TNode<Object> value, ElementsKind elements_kind, + TNode<Float64T> converted_value, TVariable<Object>* maybe_converted_value) { + Label dont_allocate_heap_number(this), end(this); + GotoIf(TaggedIsSmi(value), &dont_allocate_heap_number); + GotoIf(IsHeapNumber(CAST(value)), &dont_allocate_heap_number); + { + *maybe_converted_value = AllocateHeapNumberWithValue(converted_value); + Goto(&end); + } + BIND(&dont_allocate_heap_number); + { + *maybe_converted_value = value; + Goto(&end); + } + BIND(&end); +} + +template <> +void CodeStubAssembler::EmitElementStoreTypedArrayUpdateValue( + TNode<Object> value, ElementsKind elements_kind, + TNode<BigInt> converted_value, TVariable<Object>* maybe_converted_value) { + *maybe_converted_value = converted_value; +} + +template <typename TValue> +void CodeStubAssembler::EmitElementStoreTypedArray( + TNode<JSTypedArray> typed_array, TNode<IntPtrT> key, TNode<Object> value, + ElementsKind elements_kind, KeyedAccessStoreMode store_mode, Label* bailout, + TNode<Context> context, TVariable<Object>* maybe_converted_value) { + Label done(this), update_value_and_bailout(this, Label::kDeferred); + + TNode<TValue> converted_value = + PrepareValueForWriteToTypedArray<TValue>(value, elements_kind, context); + + // There must be no allocations between the buffer load and + // and the actual store to backing store, because GC may decide that + // the buffer is not alive or move the elements. + // TODO(ishell): introduce DisallowGarbageCollectionCode scope here. + + // Check if buffer has been detached. + TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(typed_array); + if (maybe_converted_value) { + GotoIf(IsDetachedBuffer(buffer), &update_value_and_bailout); + } else { + GotoIf(IsDetachedBuffer(buffer), bailout); + } + + // Bounds check. + TNode<UintPtrT> length = LoadJSTypedArrayLength(typed_array); + + if (store_mode == STORE_IGNORE_OUT_OF_BOUNDS) { + // Skip the store if we write beyond the length or + // to a property with a negative integer index. + GotoIfNot(UintPtrLessThan(key, length), &done); + } else { + DCHECK_EQ(store_mode, STANDARD_STORE); + GotoIfNot(UintPtrLessThan(key, length), &update_value_and_bailout); + } + + TNode<RawPtrT> data_ptr = LoadJSTypedArrayDataPtr(typed_array); + StoreElement(data_ptr, elements_kind, key, converted_value); + Goto(&done); + + BIND(&update_value_and_bailout); + // We already prepared the incoming value for storing into a typed array. + // This might involve calling ToNumber in some cases. We shouldn't call + // ToNumber again in the runtime so pass the converted value to the runtime. + // The prepared value is an untagged value. Convert it to a tagged value + // to pass it to runtime. It is not possible to do the detached buffer check + // before we prepare the value, since ToNumber can detach the ArrayBuffer. + // The spec specifies the order of these operations. + if (maybe_converted_value != nullptr) { + EmitElementStoreTypedArrayUpdateValue(value, elements_kind, converted_value, + maybe_converted_value); + } + Goto(bailout); + + BIND(&done); +} + void CodeStubAssembler::EmitElementStore( TNode<JSObject> object, TNode<Object> key, TNode<Object> value, ElementsKind elements_kind, KeyedAccessStoreMode store_mode, Label* bailout, @@ -10339,111 +10826,38 @@ void CodeStubAssembler::EmitElementStore( // TODO(rmcilroy): TNodify the converted value once this funciton and // StoreElement are templated based on the type elements_kind type. if (IsTypedArrayElementsKind(elements_kind)) { - Label done(this), update_value_and_bailout(this, Label::kDeferred); - - // IntegerIndexedElementSet converts value to a Number/BigInt prior to the - // bounds check. - Node* converted_value = - PrepareValueForWriteToTypedArray(value, elements_kind, context); TNode<JSTypedArray> typed_array = CAST(object); - - // There must be no allocations between the buffer load and - // and the actual store to backing store, because GC may decide that - // the buffer is not alive or move the elements. - // TODO(ishell): introduce DisallowGarbageCollectionCode scope here. - - // Check if buffer has been detached. - TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(typed_array); - if (maybe_converted_value) { - GotoIf(IsDetachedBuffer(buffer), &update_value_and_bailout); - } else { - GotoIf(IsDetachedBuffer(buffer), bailout); - } - - // Bounds check. - TNode<UintPtrT> length = LoadJSTypedArrayLength(typed_array); - - if (store_mode == STORE_IGNORE_OUT_OF_BOUNDS) { - // Skip the store if we write beyond the length or - // to a property with a negative integer index. - GotoIfNot(UintPtrLessThan(intptr_key, length), &done); - } else { - DCHECK_EQ(store_mode, STANDARD_STORE); - GotoIfNot(UintPtrLessThan(intptr_key, length), &update_value_and_bailout); - } - - TNode<RawPtrT> data_ptr = LoadJSTypedArrayDataPtr(typed_array); - StoreElement(data_ptr, elements_kind, intptr_key, converted_value); - Goto(&done); - - BIND(&update_value_and_bailout); - // We already prepared the incoming value for storing into a typed array. - // This might involve calling ToNumber in some cases. We shouldn't call - // ToNumber again in the runtime so pass the converted value to the runtime. - // The prepared value is an untagged value. Convert it to a tagged value - // to pass it to runtime. It is not possible to do the detached buffer check - // before we prepare the value, since ToNumber can detach the ArrayBuffer. - // The spec specifies the order of these operations. - if (maybe_converted_value != nullptr) { - switch (elements_kind) { - case UINT8_ELEMENTS: - case INT8_ELEMENTS: - case UINT16_ELEMENTS: - case INT16_ELEMENTS: - case UINT8_CLAMPED_ELEMENTS: - *maybe_converted_value = SmiFromInt32(converted_value); - break; - case UINT32_ELEMENTS: - *maybe_converted_value = ChangeUint32ToTagged(converted_value); - break; - case INT32_ELEMENTS: - *maybe_converted_value = ChangeInt32ToTagged(converted_value); - break; - case FLOAT32_ELEMENTS: { - Label dont_allocate_heap_number(this), end(this); - GotoIf(TaggedIsSmi(value), &dont_allocate_heap_number); - GotoIf(IsHeapNumber(CAST(value)), &dont_allocate_heap_number); - { - *maybe_converted_value = AllocateHeapNumberWithValue( - ChangeFloat32ToFloat64(converted_value)); - Goto(&end); - } - BIND(&dont_allocate_heap_number); - { - *maybe_converted_value = value; - Goto(&end); - } - BIND(&end); - break; - } - case FLOAT64_ELEMENTS: { - Label dont_allocate_heap_number(this), end(this); - GotoIf(TaggedIsSmi(value), &dont_allocate_heap_number); - GotoIf(IsHeapNumber(CAST(value)), &dont_allocate_heap_number); - { - *maybe_converted_value = - AllocateHeapNumberWithValue(converted_value); - Goto(&end); - } - BIND(&dont_allocate_heap_number); - { - *maybe_converted_value = value; - Goto(&end); - } - BIND(&end); - break; - } - case BIGINT64_ELEMENTS: - case BIGUINT64_ELEMENTS: - *maybe_converted_value = CAST(converted_value); - break; - default: - UNREACHABLE(); - } + switch (elements_kind) { + case UINT8_ELEMENTS: + case INT8_ELEMENTS: + case UINT16_ELEMENTS: + case INT16_ELEMENTS: + case UINT32_ELEMENTS: + case INT32_ELEMENTS: + case UINT8_CLAMPED_ELEMENTS: + EmitElementStoreTypedArray<Word32T>(typed_array, intptr_key, value, + elements_kind, store_mode, bailout, + context, maybe_converted_value); + break; + case FLOAT32_ELEMENTS: + EmitElementStoreTypedArray<Float32T>(typed_array, intptr_key, value, + elements_kind, store_mode, bailout, + context, maybe_converted_value); + break; + case FLOAT64_ELEMENTS: + EmitElementStoreTypedArray<Float64T>(typed_array, intptr_key, value, + elements_kind, store_mode, bailout, + context, maybe_converted_value); + break; + case BIGINT64_ELEMENTS: + case BIGUINT64_ELEMENTS: + EmitElementStoreTypedArray<BigInt>(typed_array, intptr_key, value, + elements_kind, store_mode, bailout, + context, maybe_converted_value); + break; + default: + UNREACHABLE(); } - Goto(bailout); - - BIND(&done); return; } DCHECK(IsFastElementsKind(elements_kind) || @@ -12559,11 +12973,6 @@ TNode<Oddball> CodeStubAssembler::HasProperty(TNode<Context> context, Label call_runtime(this, Label::kDeferred), return_true(this), return_false(this), end(this), if_proxy(this, Label::kDeferred); - if (V8_DICT_MODE_PROTOTYPES_BOOL) { - // TODO(v8:11167) remove once OrderedNameDictionary supported. - GotoIf(Int32TrueConstant(), &call_runtime); - } - CodeStubAssembler::LookupPropertyInHolder lookup_property_in_holder = [this, &return_true]( TNode<HeapObject> receiver, TNode<HeapObject> holder, @@ -13244,8 +13653,7 @@ TNode<RawPtrT> CodeStubArguments::AtIndexPtr(TNode<IntPtrT> index) const { TNode<Object> CodeStubArguments::AtIndex(TNode<IntPtrT> index) const { CSA_ASSERT(assembler_, assembler_->UintPtrOrSmiLessThan(index, GetLength())); - return assembler_->UncheckedCast<Object>( - assembler_->LoadFullTagged(AtIndexPtr(index))); + return assembler_->LoadFullTagged(AtIndexPtr(index)); } TNode<Object> CodeStubArguments::AtIndex(int index) const { @@ -13391,56 +13799,35 @@ TNode<BoolT> CodeStubAssembler::IsDebugActive() { return Word32NotEqual(is_debug_active, Int32Constant(0)); } +TNode<BoolT> CodeStubAssembler::IsPromiseHookEnabled() { + const TNode<RawPtrT> promise_hook = Load<RawPtrT>( + ExternalConstant(ExternalReference::promise_hook_address(isolate()))); + return WordNotEqual(promise_hook, IntPtrConstant(0)); +} + TNode<BoolT> CodeStubAssembler::HasAsyncEventDelegate() { const TNode<RawPtrT> async_event_delegate = Load<RawPtrT>(ExternalConstant( ExternalReference::async_event_delegate_address(isolate()))); return WordNotEqual(async_event_delegate, IntPtrConstant(0)); } -TNode<Uint32T> CodeStubAssembler::PromiseHookFlags() { - return Load<Uint32T>(ExternalConstant( - ExternalReference::promise_hook_flags_address(isolate()))); -} - -TNode<BoolT> CodeStubAssembler::IsAnyPromiseHookEnabled(TNode<Uint32T> flags) { - uint32_t mask = Isolate::PromiseHookFields::HasContextPromiseHook::kMask | - Isolate::PromiseHookFields::HasIsolatePromiseHook::kMask; - return IsSetWord32(flags, mask); -} - -TNode<BoolT> CodeStubAssembler::IsContextPromiseHookEnabled( - TNode<Uint32T> flags) { - return IsSetWord32<Isolate::PromiseHookFields::HasContextPromiseHook>(flags); -} - -TNode<BoolT> CodeStubAssembler:: - IsIsolatePromiseHookEnabledOrHasAsyncEventDelegate(TNode<Uint32T> flags) { - uint32_t mask = Isolate::PromiseHookFields::HasIsolatePromiseHook::kMask | - Isolate::PromiseHookFields::HasAsyncEventDelegate::kMask; - return IsSetWord32(flags, mask); +TNode<BoolT> CodeStubAssembler::IsPromiseHookEnabledOrHasAsyncEventDelegate() { + const TNode<Uint8T> promise_hook_or_async_event_delegate = + Load<Uint8T>(ExternalConstant( + ExternalReference::promise_hook_or_async_event_delegate_address( + isolate()))); + return Word32NotEqual(promise_hook_or_async_event_delegate, Int32Constant(0)); } TNode<BoolT> CodeStubAssembler:: - IsIsolatePromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate( - TNode<Uint32T> flags) { - uint32_t mask = Isolate::PromiseHookFields::HasIsolatePromiseHook::kMask | - Isolate::PromiseHookFields::HasAsyncEventDelegate::kMask | - Isolate::PromiseHookFields::IsDebugActive::kMask; - return IsSetWord32(flags, mask); -} - -TNode<BoolT> CodeStubAssembler:: - IsAnyPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate( - TNode<Uint32T> flags) { - return Word32NotEqual(flags, Int32Constant(0)); -} - -TNode<BoolT> CodeStubAssembler::NeedsAnyPromiseHooks(TNode<Uint32T> flags) { - uint32_t mask = Isolate::PromiseHookFields::HasContextPromiseHook::kMask | - Isolate::PromiseHookFields::HasIsolatePromiseHook::kMask | - Isolate::PromiseHookFields::HasAsyncEventDelegate::kMask | - Isolate::PromiseHookFields::IsDebugActive::kMask; - return IsSetWord32(flags, mask); + IsPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate() { + const TNode<Uint8T> promise_hook_or_debug_is_active_or_async_event_delegate = + Load<Uint8T>(ExternalConstant( + ExternalReference:: + promise_hook_or_debug_is_active_or_async_event_delegate_address( + isolate()))); + return Word32NotEqual(promise_hook_or_debug_is_active_or_async_event_delegate, + Int32Constant(0)); } TNode<Code> CodeStubAssembler::LoadBuiltin(TNode<Smi> builtin_id) { @@ -13449,14 +13836,14 @@ TNode<Code> CodeStubAssembler::LoadBuiltin(TNode<Smi> builtin_id) { TNode<IntPtrT> offset = ElementOffsetFromIndex(SmiToBInt(builtin_id), SYSTEM_POINTER_ELEMENTS); - return CAST(BitcastWordToTagged( - Load(MachineType::Pointer(), - ExternalConstant(ExternalReference::builtins_address(isolate())), - offset))); + return CAST(BitcastWordToTagged(Load<RawPtrT>( + ExternalConstant(ExternalReference::builtins_address(isolate())), + offset))); } TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( - TNode<SharedFunctionInfo> shared_info, Label* if_compile_lazy) { + TNode<SharedFunctionInfo> shared_info, TVariable<Uint16T>* data_type_out, + Label* if_compile_lazy) { TNode<Object> sfi_data = LoadObjectField(shared_info, SharedFunctionInfo::kFunctionDataOffset); @@ -13467,6 +13854,9 @@ TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( // IsSmi: Is builtin GotoIf(TaggedIsNotSmi(sfi_data), &check_instance_type); + if (data_type_out) { + *data_type_out = Uint16Constant(0); + } if (if_compile_lazy) { GotoIf(SmiEqual(CAST(sfi_data), SmiConstant(Builtins::kCompileLazy)), if_compile_lazy); @@ -13477,16 +13867,23 @@ TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( // Switch on data's instance type. BIND(&check_instance_type); TNode<Uint16T> data_type = LoadInstanceType(CAST(sfi_data)); - - int32_t case_values[] = {BYTECODE_ARRAY_TYPE, - BASELINE_DATA_TYPE, - WASM_EXPORTED_FUNCTION_DATA_TYPE, - ASM_WASM_DATA_TYPE, - UNCOMPILED_DATA_WITHOUT_PREPARSE_DATA_TYPE, - UNCOMPILED_DATA_WITH_PREPARSE_DATA_TYPE, - FUNCTION_TEMPLATE_INFO_TYPE, - WASM_JS_FUNCTION_DATA_TYPE, - WASM_CAPI_FUNCTION_DATA_TYPE}; + if (data_type_out) { + *data_type_out = data_type; + } + + int32_t case_values[] = { + BYTECODE_ARRAY_TYPE, + BASELINE_DATA_TYPE, + UNCOMPILED_DATA_WITHOUT_PREPARSE_DATA_TYPE, + UNCOMPILED_DATA_WITH_PREPARSE_DATA_TYPE, + FUNCTION_TEMPLATE_INFO_TYPE, +#if V8_ENABLE_WEBASSEMBLY + WASM_EXPORTED_FUNCTION_DATA_TYPE, + ASM_WASM_DATA_TYPE, + WASM_JS_FUNCTION_DATA_TYPE, + WASM_CAPI_FUNCTION_DATA_TYPE, +#endif // V8_ENABLE_WEBASSEMBLY + }; Label check_is_bytecode_array(this); Label check_is_baseline_data(this); Label check_is_exported_function_data(this); @@ -13497,15 +13894,19 @@ TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( Label check_is_interpreter_data(this); Label check_is_wasm_js_function_data(this); Label check_is_wasm_capi_function_data(this); - Label* case_labels[] = {&check_is_bytecode_array, - &check_is_baseline_data, - &check_is_exported_function_data, - &check_is_asm_wasm_data, - &check_is_uncompiled_data_without_preparse_data, - &check_is_uncompiled_data_with_preparse_data, - &check_is_function_template_info, - &check_is_wasm_js_function_data, - &check_is_wasm_capi_function_data}; + Label* case_labels[] = { + &check_is_bytecode_array, + &check_is_baseline_data, + &check_is_uncompiled_data_without_preparse_data, + &check_is_uncompiled_data_with_preparse_data, + &check_is_function_template_info, +#if V8_ENABLE_WEBASSEMBLY + &check_is_exported_function_data, + &check_is_asm_wasm_data, + &check_is_wasm_js_function_data, + &check_is_wasm_capi_function_data +#endif // V8_ENABLE_WEBASSEMBLY + }; STATIC_ASSERT(arraysize(case_values) == arraysize(case_labels)); Switch(data_type, &check_is_interpreter_data, case_values, case_labels, arraysize(case_labels)); @@ -13523,17 +13924,6 @@ TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( sfi_code = baseline_code; Goto(&done); - // IsWasmExportedFunctionData: Use the wrapper code - BIND(&check_is_exported_function_data); - sfi_code = CAST(LoadObjectField( - CAST(sfi_data), WasmExportedFunctionData::kWrapperCodeOffset)); - Goto(&done); - - // IsAsmWasmData: Instantiate using AsmWasmData - BIND(&check_is_asm_wasm_data); - sfi_code = HeapConstant(BUILTIN_CODE(isolate(), InstantiateAsmJs)); - Goto(&done); - // IsUncompiledDataWithPreparseData | IsUncompiledDataWithoutPreparseData: // Compile lazy BIND(&check_is_uncompiled_data_with_preparse_data); @@ -13556,6 +13946,18 @@ TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( CAST(sfi_data), InterpreterData::kInterpreterTrampolineOffset)); Goto(&done); +#if V8_ENABLE_WEBASSEMBLY + // IsWasmExportedFunctionData: Use the wrapper code + BIND(&check_is_exported_function_data); + sfi_code = CAST(LoadObjectField( + CAST(sfi_data), WasmExportedFunctionData::kWrapperCodeOffset)); + Goto(&done); + + // IsAsmWasmData: Instantiate using AsmWasmData + BIND(&check_is_asm_wasm_data); + sfi_code = HeapConstant(BUILTIN_CODE(isolate(), InstantiateAsmJs)); + Goto(&done); + // IsWasmJSFunctionData: Use the wrapper code. BIND(&check_is_wasm_js_function_data); sfi_code = CAST( @@ -13567,6 +13969,7 @@ TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( sfi_code = CAST(LoadObjectField(CAST(sfi_data), WasmCapiFunctionData::kWrapperCodeOffset)); Goto(&done); +#endif // V8_ENABLE_WEBASSEMBLY BIND(&done); return sfi_code.value(); @@ -13661,15 +14064,15 @@ TNode<Map> CodeStubAssembler::CheckEnumCache(TNode<JSReceiver> receiver, TNode<Smi> length; TNode<HeapObject> properties = LoadSlowProperties(receiver); - if (V8_DICT_MODE_PROTOTYPES_BOOL) { - CSA_ASSERT(this, Word32Or(IsOrderedNameDictionary(properties), + if (V8_ENABLE_SWISS_NAME_DICTIONARY_BOOL) { + CSA_ASSERT(this, Word32Or(IsSwissNameDictionary(properties), IsGlobalDictionary(properties))); length = Select<Smi>( - IsOrderedNameDictionary(properties), + IsSwissNameDictionary(properties), [=] { return GetNumberOfElements( - UncheckedCast<OrderedNameDictionary>(properties)); + UncheckedCast<SwissNameDictionary>(properties)); }, [=] { return GetNumberOfElements( @@ -14039,5 +14442,729 @@ void PrototypeCheckAssembler::CheckAndBranch(TNode<HeapObject> prototype, } } +// +// Begin of SwissNameDictionary macros +// + +namespace { + +// Provides load and store functions that abstract over the details of accessing +// the meta table in memory. Instead they allow using logical indices that are +// independent from the underlying entry size in the meta table of a +// SwissNameDictionary. +class MetaTableAccessor { + public: + MetaTableAccessor(CodeStubAssembler& csa, MachineType mt) + : csa{csa}, mt{mt} {} + + TNode<Uint32T> Load(TNode<ByteArray> meta_table, TNode<IntPtrT> index) { + TNode<IntPtrT> offset = OverallOffset(meta_table, index); + + return csa.UncheckedCast<Uint32T>( + csa.LoadFromObject(mt, meta_table, offset)); + } + + TNode<Uint32T> Load(TNode<ByteArray> meta_table, int index) { + return Load(meta_table, csa.IntPtrConstant(index)); + } + + void Store(TNode<ByteArray> meta_table, TNode<IntPtrT> index, + TNode<Uint32T> data) { + TNode<IntPtrT> offset = OverallOffset(meta_table, index); + +#ifdef DEBUG + int bits = mt.MemSize() * 8; + TNode<UintPtrT> max_value = csa.UintPtrConstant((1ULL << bits) - 1); + + CSA_ASSERT(&csa, csa.UintPtrLessThanOrEqual(csa.ChangeUint32ToWord(data), + max_value)); +#endif + + csa.StoreToObject(mt.representation(), meta_table, offset, data, + StoreToObjectWriteBarrier::kNone); + } + + void Store(TNode<ByteArray> meta_table, int index, TNode<Uint32T> data) { + Store(meta_table, csa.IntPtrConstant(index), data); + } + + private: + TNode<IntPtrT> OverallOffset(TNode<ByteArray> meta_table, + TNode<IntPtrT> index) { + // TODO(v8:11330): consider using ElementOffsetFromIndex(). + + int offset_to_data_minus_tag = ByteArray::kHeaderSize - kHeapObjectTag; + + TNode<IntPtrT> overall_offset; + int size = mt.MemSize(); + intptr_t constant; + if (csa.TryToIntPtrConstant(index, &constant)) { + intptr_t index_offset = constant * size; + overall_offset = + csa.IntPtrConstant(offset_to_data_minus_tag + index_offset); + } else { + TNode<IntPtrT> index_offset = + csa.IntPtrMul(index, csa.IntPtrConstant(size)); + overall_offset = csa.IntPtrAdd( + csa.IntPtrConstant(offset_to_data_minus_tag), index_offset); + } + +#ifdef DEBUG + TNode<IntPtrT> byte_array_data_bytes = + csa.SmiToIntPtr(csa.LoadFixedArrayBaseLength(meta_table)); + TNode<IntPtrT> max_allowed_offset = csa.IntPtrAdd( + byte_array_data_bytes, csa.IntPtrConstant(offset_to_data_minus_tag)); + CSA_ASSERT(&csa, csa.UintPtrLessThan(overall_offset, max_allowed_offset)); +#endif + + return overall_offset; + } + + CodeStubAssembler& csa; + MachineType mt; +}; + +// Type of functions that given a MetaTableAccessor, use its load and store +// functions to generate code for operating on the meta table. +using MetaTableAccessFunction = std::function<void(MetaTableAccessor&)>; + +// Helper function for macros operating on the meta table of a +// SwissNameDictionary. Given a MetaTableAccessFunction, generates branching +// code and uses the builder to generate code for each of the three possible +// sizes per entry a meta table can have. +void GenerateMetaTableAccess(CodeStubAssembler* csa, TNode<IntPtrT> capacity, + MetaTableAccessFunction builder) { + MetaTableAccessor mta8 = MetaTableAccessor(*csa, MachineType::Uint8()); + MetaTableAccessor mta16 = MetaTableAccessor(*csa, MachineType::Uint16()); + MetaTableAccessor mta32 = MetaTableAccessor(*csa, MachineType::Uint32()); + + using Label = compiler::CodeAssemblerLabel; + Label small(csa), medium(csa), done(csa); + + csa->GotoIf( + csa->IntPtrLessThanOrEqual( + capacity, + csa->IntPtrConstant(SwissNameDictionary::kMax1ByteMetaTableCapacity)), + &small); + csa->GotoIf( + csa->IntPtrLessThanOrEqual( + capacity, + csa->IntPtrConstant(SwissNameDictionary::kMax2ByteMetaTableCapacity)), + &medium); + + builder(mta32); + csa->Goto(&done); + + csa->Bind(&medium); + builder(mta16); + csa->Goto(&done); + + csa->Bind(&small); + builder(mta8); + csa->Goto(&done); + csa->Bind(&done); +} + +} // namespace + +TNode<IntPtrT> CodeStubAssembler::LoadSwissNameDictionaryNumberOfElements( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity) { + TNode<ByteArray> meta_table = LoadSwissNameDictionaryMetaTable(table); + + TVARIABLE(Uint32T, nof, Uint32Constant(0)); + MetaTableAccessFunction builder = [&](MetaTableAccessor& mta) { + nof = mta.Load(meta_table, + SwissNameDictionary::kMetaTableElementCountFieldIndex); + }; + + GenerateMetaTableAccess(this, capacity, builder); + return ChangeInt32ToIntPtr(nof.value()); +} + +TNode<IntPtrT> +CodeStubAssembler::LoadSwissNameDictionaryNumberOfDeletedElements( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity) { + TNode<ByteArray> meta_table = LoadSwissNameDictionaryMetaTable(table); + + TVARIABLE(Uint32T, nod, Uint32Constant(0)); + MetaTableAccessFunction builder = [&](MetaTableAccessor& mta) { + nod = + mta.Load(meta_table, + SwissNameDictionary::kMetaTableDeletedElementCountFieldIndex); + }; + + GenerateMetaTableAccess(this, capacity, builder); + return ChangeInt32ToIntPtr(nod.value()); +} + +void CodeStubAssembler::StoreSwissNameDictionaryEnumToEntryMapping( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity, + TNode<IntPtrT> enum_index, TNode<Int32T> entry) { + TNode<ByteArray> meta_table = LoadSwissNameDictionaryMetaTable(table); + TNode<IntPtrT> meta_table_index = IntPtrAdd( + IntPtrConstant(SwissNameDictionary::kMetaTableEnumerationDataStartIndex), + enum_index); + + MetaTableAccessFunction builder = [&](MetaTableAccessor& mta) { + mta.Store(meta_table, meta_table_index, Unsigned(entry)); + }; + + GenerateMetaTableAccess(this, capacity, builder); +} + +TNode<Uint32T> +CodeStubAssembler::SwissNameDictionaryIncreaseElementCountOrBailout( + TNode<ByteArray> meta_table, TNode<IntPtrT> capacity, + TNode<Uint32T> max_usable_capacity, Label* bailout) { + TVARIABLE(Uint32T, used_var, Uint32Constant(0)); + + MetaTableAccessFunction builder = [&](MetaTableAccessor& mta) { + TNode<Uint32T> nof = mta.Load( + meta_table, SwissNameDictionary::kMetaTableElementCountFieldIndex); + TNode<Uint32T> nod = + mta.Load(meta_table, + SwissNameDictionary::kMetaTableDeletedElementCountFieldIndex); + TNode<Uint32T> used = Uint32Add(nof, nod); + GotoIf(Uint32GreaterThanOrEqual(used, max_usable_capacity), bailout); + TNode<Uint32T> inc_nof = Uint32Add(nof, Uint32Constant(1)); + mta.Store(meta_table, SwissNameDictionary::kMetaTableElementCountFieldIndex, + inc_nof); + used_var = used; + }; + + GenerateMetaTableAccess(this, capacity, builder); + return used_var.value(); +} + +TNode<Uint32T> CodeStubAssembler::SwissNameDictionaryUpdateCountsForDeletion( + TNode<ByteArray> meta_table, TNode<IntPtrT> capacity) { + TVARIABLE(Uint32T, new_nof_var, Uint32Constant(0)); + + MetaTableAccessFunction builder = [&](MetaTableAccessor& mta) { + TNode<Uint32T> nof = mta.Load( + meta_table, SwissNameDictionary::kMetaTableElementCountFieldIndex); + TNode<Uint32T> nod = + mta.Load(meta_table, + SwissNameDictionary::kMetaTableDeletedElementCountFieldIndex); + + TNode<Uint32T> new_nof = Uint32Sub(nof, Uint32Constant(1)); + TNode<Uint32T> new_nod = Uint32Add(nod, Uint32Constant(1)); + + mta.Store(meta_table, SwissNameDictionary::kMetaTableElementCountFieldIndex, + new_nof); + mta.Store(meta_table, + SwissNameDictionary::kMetaTableDeletedElementCountFieldIndex, + new_nod); + + new_nof_var = new_nof; + }; + + GenerateMetaTableAccess(this, capacity, builder); + return new_nof_var.value(); +} + +TNode<SwissNameDictionary> CodeStubAssembler::AllocateSwissNameDictionary( + TNode<IntPtrT> at_least_space_for) { + // Note that as AllocateNameDictionary, we return a table with initial + // (non-zero) capacity even if |at_least_space_for| is 0. + + TNode<IntPtrT> capacity = + IntPtrMax(IntPtrConstant(SwissNameDictionary::kInitialCapacity), + SwissNameDictionaryCapacityFor(at_least_space_for)); + + return AllocateSwissNameDictionaryWithCapacity(capacity); +} + +TNode<SwissNameDictionary> CodeStubAssembler::AllocateSwissNameDictionary( + int at_least_space_for) { + return AllocateSwissNameDictionary(IntPtrConstant(at_least_space_for)); +} + +TNode<SwissNameDictionary> +CodeStubAssembler::AllocateSwissNameDictionaryWithCapacity( + TNode<IntPtrT> capacity) { + Comment("[ AllocateSwissNameDictionaryWithCapacity"); + CSA_ASSERT(this, WordIsPowerOfTwo(capacity)); + CSA_ASSERT(this, UintPtrGreaterThanOrEqual( + capacity, + IntPtrConstant(SwissNameDictionary::kInitialCapacity))); + CSA_ASSERT(this, + UintPtrLessThanOrEqual( + capacity, IntPtrConstant(SwissNameDictionary::MaxCapacity()))); + + Comment("Size check."); + intptr_t capacity_constant; + if (ToParameterConstant(capacity, &capacity_constant)) { + CHECK_LE(capacity_constant, SwissNameDictionary::MaxCapacity()); + } else { + Label if_out_of_memory(this, Label::kDeferred), next(this); + Branch(UintPtrGreaterThan( + capacity, IntPtrConstant(SwissNameDictionary::MaxCapacity())), + &if_out_of_memory, &next); + + BIND(&if_out_of_memory); + CallRuntime(Runtime::kFatalProcessOutOfMemoryInAllocateRaw, + NoContextConstant()); + Unreachable(); + + BIND(&next); + } + + // TODO(v8:11330) Consider adding dedicated handling for constant capacties, + // similar to AllocateOrderedHashTableWithCapacity. + + // We must allocate the ByteArray first. Otherwise, allocating the ByteArray + // may trigger GC, which may try to verify the un-initialized + // SwissNameDictionary. + Comment("Meta table allocation."); + TNode<IntPtrT> meta_table_payload_size = + SwissNameDictionaryMetaTableSizeFor(capacity); + + TNode<ByteArray> meta_table = + AllocateNonEmptyByteArray(Unsigned(meta_table_payload_size), + AllocationFlag::kAllowLargeObjectAllocation); + + Comment("SwissNameDictionary allocation."); + TNode<IntPtrT> total_size = SwissNameDictionarySizeFor(capacity); + + TNode<SwissNameDictionary> table = UncheckedCast<SwissNameDictionary>( + Allocate(total_size, kAllowLargeObjectAllocation)); + + StoreMapNoWriteBarrier(table, RootIndex::kSwissNameDictionaryMap); + + Comment( + "Initialize the hash, capacity, meta table pointer, and number of " + "(deleted) elements."); + + StoreSwissNameDictionaryHash(table, + Uint32Constant(PropertyArray::kNoHashSentinel)); + StoreSwissNameDictionaryCapacity(table, TruncateIntPtrToInt32(capacity)); + StoreSwissNameDictionaryMetaTable(table, meta_table); + + // Set present and deleted element count without doing branching needed for + // meta table access twice. + MetaTableAccessFunction builder = [&](MetaTableAccessor& mta) { + mta.Store(meta_table, SwissNameDictionary::kMetaTableElementCountFieldIndex, + Uint32Constant(0)); + mta.Store(meta_table, + SwissNameDictionary::kMetaTableDeletedElementCountFieldIndex, + Uint32Constant(0)); + }; + GenerateMetaTableAccess(this, capacity, builder); + + Comment("Initialize the ctrl table."); + + TNode<IntPtrT> ctrl_table_start_offset_minus_tag = + SwissNameDictionaryCtrlTableStartOffsetMT(capacity); + + TNode<IntPtrT> table_address_with_tag = BitcastTaggedToWord(table); + TNode<IntPtrT> ctrl_table_size_bytes = + IntPtrAdd(capacity, IntPtrConstant(SwissNameDictionary::kGroupWidth)); + TNode<IntPtrT> ctrl_table_start_ptr = + IntPtrAdd(table_address_with_tag, ctrl_table_start_offset_minus_tag); + TNode<IntPtrT> ctrl_table_end_ptr = + IntPtrAdd(ctrl_table_start_ptr, ctrl_table_size_bytes); + + // |ctrl_table_size_bytes| (= capacity + kGroupWidth) is divisble by four: + STATIC_ASSERT(SwissNameDictionary::kGroupWidth % 4 == 0); + STATIC_ASSERT(SwissNameDictionary::kInitialCapacity % 4 == 0); + + // TODO(v8:11330) For all capacities except 4, we know that + // |ctrl_table_size_bytes| is divisible by 8. Consider initializing the ctrl + // table with WordTs in those cases. Alternatively, always initialize as many + // bytes as possbible with WordT and then, if necessary, the remaining 4 bytes + // with Word32T. + + constexpr uint8_t kEmpty = swiss_table::Ctrl::kEmpty; + constexpr uint32_t kEmpty32 = + (kEmpty << 24) | (kEmpty << 16) | (kEmpty << 8) | kEmpty; + TNode<Int32T> empty32 = Int32Constant(kEmpty32); + BuildFastLoop<IntPtrT>( + ctrl_table_start_ptr, ctrl_table_end_ptr, + [=](TNode<IntPtrT> current) { + UnsafeStoreNoWriteBarrier(MachineRepresentation::kWord32, current, + empty32); + }, + sizeof(uint32_t), IndexAdvanceMode::kPost); + + Comment("Initialize the data table."); + + TNode<IntPtrT> data_table_start_offset_minus_tag = + SwissNameDictionaryDataTableStartOffsetMT(); + TNode<IntPtrT> data_table_ptr = + IntPtrAdd(table_address_with_tag, data_table_start_offset_minus_tag); + TNode<IntPtrT> data_table_size = IntPtrMul( + IntPtrConstant(SwissNameDictionary::kDataTableEntryCount * kTaggedSize), + capacity); + + StoreFieldsNoWriteBarrier(data_table_ptr, + IntPtrAdd(data_table_ptr, data_table_size), + TheHoleConstant()); + + Comment("AllocateSwissNameDictionaryWithCapacity ]"); + + return table; +} + +TNode<SwissNameDictionary> CodeStubAssembler::CopySwissNameDictionary( + TNode<SwissNameDictionary> original) { + Comment("[ CopySwissNameDictionary"); + + TNode<IntPtrT> capacity = + Signed(ChangeUint32ToWord(LoadSwissNameDictionaryCapacity(original))); + + // We must allocate the ByteArray first. Otherwise, allocating the ByteArray + // may trigger GC, which may try to verify the un-initialized + // SwissNameDictionary. + Comment("Meta table allocation."); + TNode<IntPtrT> meta_table_payload_size = + SwissNameDictionaryMetaTableSizeFor(capacity); + + TNode<ByteArray> meta_table = + AllocateNonEmptyByteArray(Unsigned(meta_table_payload_size), + AllocationFlag::kAllowLargeObjectAllocation); + + Comment("SwissNameDictionary allocation."); + TNode<IntPtrT> total_size = SwissNameDictionarySizeFor(capacity); + + TNode<SwissNameDictionary> table = UncheckedCast<SwissNameDictionary>( + Allocate(total_size, kAllowLargeObjectAllocation)); + + StoreMapNoWriteBarrier(table, RootIndex::kSwissNameDictionaryMap); + + Comment("Copy the hash and capacity."); + + StoreSwissNameDictionaryHash(table, LoadSwissNameDictionaryHash(original)); + StoreSwissNameDictionaryCapacity(table, TruncateIntPtrToInt32(capacity)); + StoreSwissNameDictionaryMetaTable(table, meta_table); + // Not setting up number of (deleted elements), copying whole meta table + // instead. + + TNode<ExternalReference> memcpy = + ExternalConstant(ExternalReference::libc_memcpy_function()); + + TNode<IntPtrT> old_table_address_with_tag = BitcastTaggedToWord(original); + TNode<IntPtrT> new_table_address_with_tag = BitcastTaggedToWord(table); + + TNode<IntPtrT> ctrl_table_start_offset_minus_tag = + SwissNameDictionaryCtrlTableStartOffsetMT(capacity); + + TNode<IntPtrT> ctrl_table_size_bytes = + IntPtrAdd(capacity, IntPtrConstant(SwissNameDictionary::kGroupWidth)); + + Comment("Copy the ctrl table."); + { + TNode<IntPtrT> old_ctrl_table_start_ptr = IntPtrAdd( + old_table_address_with_tag, ctrl_table_start_offset_minus_tag); + TNode<IntPtrT> new_ctrl_table_start_ptr = IntPtrAdd( + new_table_address_with_tag, ctrl_table_start_offset_minus_tag); + + CallCFunction( + memcpy, MachineType::Pointer(), + std::make_pair(MachineType::Pointer(), new_ctrl_table_start_ptr), + std::make_pair(MachineType::Pointer(), old_ctrl_table_start_ptr), + std::make_pair(MachineType::UintPtr(), ctrl_table_size_bytes)); + } + + Comment("Copy the data table."); + { + TNode<IntPtrT> start_offset = + IntPtrConstant(SwissNameDictionary::DataTableStartOffset()); + TNode<IntPtrT> data_table_size = IntPtrMul( + IntPtrConstant(SwissNameDictionary::kDataTableEntryCount * kTaggedSize), + capacity); + + BuildFastLoop<IntPtrT>( + start_offset, IntPtrAdd(start_offset, data_table_size), + [=](TNode<IntPtrT> offset) { + TNode<Object> table_field = LoadObjectField(original, offset); + StoreObjectField(table, offset, table_field); + }, + kTaggedSize, IndexAdvanceMode::kPost); + } + + Comment("Copy the meta table"); + { + TNode<IntPtrT> old_meta_table_address_with_tag = + BitcastTaggedToWord(LoadSwissNameDictionaryMetaTable(original)); + TNode<IntPtrT> new_meta_table_address_with_tag = + BitcastTaggedToWord(meta_table); + + TNode<IntPtrT> meta_table_size = + SwissNameDictionaryMetaTableSizeFor(capacity); + + TNode<IntPtrT> old_data_start = + IntPtrAdd(old_meta_table_address_with_tag, + IntPtrConstant(ByteArray::kHeaderSize - kHeapObjectTag)); + TNode<IntPtrT> new_data_start = + IntPtrAdd(new_meta_table_address_with_tag, + IntPtrConstant(ByteArray::kHeaderSize - kHeapObjectTag)); + + CallCFunction(memcpy, MachineType::Pointer(), + std::make_pair(MachineType::Pointer(), new_data_start), + std::make_pair(MachineType::Pointer(), old_data_start), + std::make_pair(MachineType::UintPtr(), meta_table_size)); + } + + Comment("Copy the PropertyDetails table"); + { + TNode<IntPtrT> property_details_start_offset_minus_tag = + SwissNameDictionaryOffsetIntoPropertyDetailsTableMT(table, capacity, + IntPtrConstant(0)); + + // Offset to property details entry + TVARIABLE(IntPtrT, details_table_offset_minus_tag, + property_details_start_offset_minus_tag); + + TNode<IntPtrT> start = ctrl_table_start_offset_minus_tag; + + VariableList in_loop_variables({&details_table_offset_minus_tag}, zone()); + BuildFastLoop<IntPtrT>( + in_loop_variables, start, IntPtrAdd(start, ctrl_table_size_bytes), + [&](TNode<IntPtrT> ctrl_table_offset) { + TNode<Uint8T> ctrl = Load<Uint8T>(original, ctrl_table_offset); + + // TODO(v8:11330) Entries in the PropertyDetails table may be + // uninitialized if the corresponding buckets in the data/ctrl table + // are empty. Therefore, to avoid accessing un-initialized memory + // here, we need to check the ctrl table to determine whether we + // should copy a certain PropertyDetails entry or not. + // TODO(v8:11330) If this function becomes performance-critical, we + // may consider always initializing the PropertyDetails table entirely + // during allocation, to avoid the branching during copying. + Label done(this); + // |kNotFullMask| catches kEmpty and kDeleted, both of which indicate + // entries that we don't want to copy the PropertyDetails for. + GotoIf(IsSetWord32(ctrl, swiss_table::kNotFullMask), &done); + + TNode<Uint8T> details = + Load<Uint8T>(original, details_table_offset_minus_tag.value()); + + StoreToObject(MachineRepresentation::kWord8, table, + details_table_offset_minus_tag.value(), details, + StoreToObjectWriteBarrier::kNone); + Goto(&done); + BIND(&done); + + details_table_offset_minus_tag = + IntPtrAdd(details_table_offset_minus_tag.value(), + IntPtrConstant(kOneByteSize)); + }, + kOneByteSize, IndexAdvanceMode::kPost); + } + + Comment("CopySwissNameDictionary ]"); + + return table; +} + +TNode<IntPtrT> CodeStubAssembler::SwissNameDictionaryOffsetIntoDataTableMT( + TNode<SwissNameDictionary> dict, TNode<IntPtrT> index, int field_index) { + TNode<IntPtrT> data_table_start = SwissNameDictionaryDataTableStartOffsetMT(); + + TNode<IntPtrT> offset_within_data_table = IntPtrMul( + index, + IntPtrConstant(SwissNameDictionary::kDataTableEntryCount * kTaggedSize)); + + if (field_index != 0) { + offset_within_data_table = IntPtrAdd( + offset_within_data_table, IntPtrConstant(field_index * kTaggedSize)); + } + + return IntPtrAdd(data_table_start, offset_within_data_table); +} + +TNode<IntPtrT> +CodeStubAssembler::SwissNameDictionaryOffsetIntoPropertyDetailsTableMT( + TNode<SwissNameDictionary> dict, TNode<IntPtrT> capacity, + TNode<IntPtrT> index) { + CSA_ASSERT(this, + WordEqual(capacity, ChangeUint32ToWord( + LoadSwissNameDictionaryCapacity(dict)))); + + TNode<IntPtrT> data_table_start = SwissNameDictionaryDataTableStartOffsetMT(); + + TNode<IntPtrT> gw = IntPtrConstant(SwissNameDictionary::kGroupWidth); + TNode<IntPtrT> data_and_ctrl_table_size = IntPtrAdd( + IntPtrMul(capacity, + IntPtrConstant(kOneByteSize + + SwissNameDictionary::kDataTableEntryCount * + kTaggedSize)), + gw); + + TNode<IntPtrT> property_details_table_start = + IntPtrAdd(data_table_start, data_and_ctrl_table_size); + + CSA_ASSERT( + this, + WordEqual(FieldSliceSwissNameDictionaryPropertyDetailsTable(dict).offset, + // Our calculation subtracted the tag, Torque's offset didn't. + IntPtrAdd(property_details_table_start, + IntPtrConstant(kHeapObjectTag)))); + + TNode<IntPtrT> offset_within_details_table = index; + return IntPtrAdd(property_details_table_start, offset_within_details_table); +} + +void CodeStubAssembler::StoreSwissNameDictionaryCapacity( + TNode<SwissNameDictionary> table, TNode<Int32T> capacity) { + StoreObjectFieldNoWriteBarrier<Word32T>( + table, SwissNameDictionary::CapacityOffset(), capacity); +} + +TNode<Name> CodeStubAssembler::LoadSwissNameDictionaryKey( + TNode<SwissNameDictionary> dict, TNode<IntPtrT> entry) { + TNode<IntPtrT> offset_minus_tag = SwissNameDictionaryOffsetIntoDataTableMT( + dict, entry, SwissNameDictionary::kDataTableKeyEntryIndex); + + // TODO(v8:11330) Consider using LoadObjectField here. + return CAST(Load<Object>(dict, offset_minus_tag)); +} + +TNode<Uint8T> CodeStubAssembler::LoadSwissNameDictionaryPropertyDetails( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity, + TNode<IntPtrT> entry) { + TNode<IntPtrT> offset_minus_tag = + SwissNameDictionaryOffsetIntoPropertyDetailsTableMT(table, capacity, + entry); + // TODO(v8:11330) Consider using LoadObjectField here. + return Load<Uint8T>(table, offset_minus_tag); +} + +void CodeStubAssembler::StoreSwissNameDictionaryPropertyDetails( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity, + TNode<IntPtrT> entry, TNode<Uint8T> details) { + TNode<IntPtrT> offset_minus_tag = + SwissNameDictionaryOffsetIntoPropertyDetailsTableMT(table, capacity, + entry); + + // TODO(v8:11330) Consider using StoreObjectField here. + StoreToObject(MachineRepresentation::kWord8, table, offset_minus_tag, details, + StoreToObjectWriteBarrier::kNone); +} + +void CodeStubAssembler::StoreSwissNameDictionaryKeyAndValue( + TNode<SwissNameDictionary> dict, TNode<IntPtrT> entry, TNode<Object> key, + TNode<Object> value) { + STATIC_ASSERT(SwissNameDictionary::kDataTableKeyEntryIndex == 0); + STATIC_ASSERT(SwissNameDictionary::kDataTableValueEntryIndex == 1); + + // TODO(v8:11330) Consider using StoreObjectField here. + TNode<IntPtrT> key_offset_minus_tag = + SwissNameDictionaryOffsetIntoDataTableMT( + dict, entry, SwissNameDictionary::kDataTableKeyEntryIndex); + StoreToObject(MachineRepresentation::kTagged, dict, key_offset_minus_tag, key, + StoreToObjectWriteBarrier::kFull); + + TNode<IntPtrT> value_offset_minus_tag = + IntPtrAdd(key_offset_minus_tag, IntPtrConstant(kTaggedSize)); + StoreToObject(MachineRepresentation::kTagged, dict, value_offset_minus_tag, + value, StoreToObjectWriteBarrier::kFull); +} + +TNode<Uint64T> CodeStubAssembler::LoadSwissNameDictionaryCtrlTableGroup( + TNode<IntPtrT> address) { + TNode<RawPtrT> ptr = ReinterpretCast<RawPtrT>(address); + TNode<Uint64T> data = UnalignedLoad<Uint64T>(ptr, IntPtrConstant(0)); + +#ifdef V8_TARGET_LITTLE_ENDIAN + return data; +#else + // Reverse byte order. + // TODO(v8:11330) Doing this without using dedicated instructions (which we + // don't have access to here) will destroy any performance benefit Swiss + // Tables have. So we just support this so that we don't have to disable the + // test suite for SwissNameDictionary on big endian platforms. + + TNode<Uint64T> result = Uint64Constant(0); + constexpr int count = sizeof(uint64_t); + for (int i = 0; i < count; ++i) { + int src_offset = i * 8; + int dest_offset = (count - i - 1) * 8; + + TNode<Uint64T> mask = Uint64Constant(0xffULL << src_offset); + TNode<Uint64T> src_data = Word64And(data, mask); + + TNode<Uint64T> shifted = + src_offset < dest_offset + ? Word64Shl(src_data, Uint64Constant(dest_offset - src_offset)) + : Word64Shr(src_data, Uint64Constant(src_offset - dest_offset)); + result = Unsigned(Word64Or(result, shifted)); + } + return result; +#endif +} + +void CodeStubAssembler::SwissNameDictionarySetCtrl( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity, + TNode<IntPtrT> entry, TNode<Uint8T> ctrl) { + CSA_ASSERT(this, + WordEqual(capacity, ChangeUint32ToWord( + LoadSwissNameDictionaryCapacity(table)))); + CSA_ASSERT(this, UintPtrLessThan(entry, capacity)); + + TNode<IntPtrT> one = IntPtrConstant(1); + TNode<IntPtrT> offset = SwissNameDictionaryCtrlTableStartOffsetMT(capacity); + + CSA_ASSERT(this, + WordEqual(FieldSliceSwissNameDictionaryCtrlTable(table).offset, + IntPtrAdd(offset, one))); + + TNode<IntPtrT> offset_entry = IntPtrAdd(offset, entry); + StoreToObject(MachineRepresentation::kWord8, table, offset_entry, ctrl, + StoreToObjectWriteBarrier::kNone); + + TNode<IntPtrT> mask = IntPtrSub(capacity, one); + TNode<IntPtrT> group_width = IntPtrConstant(SwissNameDictionary::kGroupWidth); + + // See SwissNameDictionary::SetCtrl for description of what's going on here. + + // ((entry - Group::kWidth) & mask) + 1 + TNode<IntPtrT> copy_entry_lhs = + IntPtrAdd(WordAnd(IntPtrSub(entry, group_width), mask), one); + // ((Group::kWidth - 1) & mask) + TNode<IntPtrT> copy_entry_rhs = WordAnd(IntPtrSub(group_width, one), mask); + TNode<IntPtrT> copy_entry = IntPtrAdd(copy_entry_lhs, copy_entry_rhs); + TNode<IntPtrT> offset_copy_entry = IntPtrAdd(offset, copy_entry); + + // |entry| < |kGroupWidth| implies |copy_entry| == |capacity| + |entry| + CSA_ASSERT(this, Word32Or(UintPtrGreaterThanOrEqual(entry, group_width), + WordEqual(copy_entry, IntPtrAdd(capacity, entry)))); + + // |entry| >= |kGroupWidth| implies |copy_entry| == |entry| + CSA_ASSERT(this, Word32Or(UintPtrLessThan(entry, group_width), + WordEqual(copy_entry, entry))); + + // TODO(v8:11330): consider using StoreObjectFieldNoWriteBarrier here. + StoreToObject(MachineRepresentation::kWord8, table, offset_copy_entry, ctrl, + StoreToObjectWriteBarrier::kNone); +} + +void CodeStubAssembler::SwissNameDictionaryFindEntry( + TNode<SwissNameDictionary> table, TNode<Name> key, Label* found, + TVariable<IntPtrT>* var_found_entry, Label* not_found) { + if (SwissNameDictionary::kUseSIMD) { + SwissNameDictionaryFindEntrySIMD(table, key, found, var_found_entry, + not_found); + } else { + SwissNameDictionaryFindEntryPortable(table, key, found, var_found_entry, + not_found); + } +} + +void CodeStubAssembler::SwissNameDictionaryAdd(TNode<SwissNameDictionary> table, + TNode<Name> key, + TNode<Object> value, + TNode<Uint8T> property_details, + Label* needs_resize) { + if (SwissNameDictionary::kUseSIMD) { + SwissNameDictionaryAddSIMD(table, key, value, property_details, + needs_resize); + } else { + SwissNameDictionaryAddPortable(table, key, value, property_details, + needs_resize); + } +} + } // namespace internal } // namespace v8 diff --git a/deps/v8/src/codegen/code-stub-assembler.h b/deps/v8/src/codegen/code-stub-assembler.h index ba331624c6..72b8fbc8a8 100644 --- a/deps/v8/src/codegen/code-stub-assembler.h +++ b/deps/v8/src/codegen/code-stub-assembler.h @@ -135,6 +135,8 @@ enum class PrimitiveType { kBoolean, kNumber, kString, kSymbol }; EmptyPropertyDictionary) \ V(EmptyOrderedPropertyDictionary, empty_ordered_property_dictionary, \ EmptyOrderedPropertyDictionary) \ + V(EmptySwissPropertyDictionary, empty_swiss_property_dictionary, \ + EmptySwissPropertyDictionary) \ V(EmptySlowElementDictionary, empty_slow_element_dictionary, \ EmptySlowElementDictionary) \ V(empty_string, empty_string, EmptyString) \ @@ -305,7 +307,6 @@ class V8_EXPORT_PRIVATE CodeStubAssembler : public compiler::CodeAssembler, public TorqueGeneratedExportedMacrosAssembler { public: - using Node = compiler::Node; using ScopedExceptionHandler = compiler::ScopedExceptionHandler; template <typename T> @@ -539,6 +540,14 @@ class V8_EXPORT_PRIVATE CodeStubAssembler bool TryGetIntPtrOrSmiConstantValue(TNode<IntPtrT> maybe_constant, int* value); + TNode<IntPtrT> PopulationCountFallback(TNode<UintPtrT> value); + TNode<Int64T> PopulationCount64(TNode<Word64T> value); + TNode<Int32T> PopulationCount32(TNode<Word32T> value); + TNode<Int64T> CountTrailingZeros64(TNode<Word64T> value); + TNode<Int32T> CountTrailingZeros32(TNode<Word32T> value); + TNode<Int64T> CountLeadingZeros64(TNode<Word64T> value); + TNode<Int32T> CountLeadingZeros32(TNode<Word32T> value); + // Round the 32bits payload of the provided word up to the next power of two. TNode<IntPtrT> IntPtrRoundUpToPowerOfTwo32(TNode<IntPtrT> value); // Select the maximum of the two provided IntPtr values. @@ -569,7 +578,7 @@ class V8_EXPORT_PRIVATE CodeStubAssembler // Smi conversions. TNode<Float64T> SmiToFloat64(TNode<Smi> value); TNode<Smi> SmiFromIntPtr(TNode<IntPtrT> value) { return SmiTag(value); } - TNode<Smi> SmiFromInt32(SloppyTNode<Int32T> value); + TNode<Smi> SmiFromInt32(TNode<Int32T> value); TNode<Smi> SmiFromUint32(TNode<Uint32T> value); TNode<IntPtrT> SmiToIntPtr(TNode<Smi> value) { return SmiUntag(value); } TNode<Int32T> SmiToInt32(TNode<Smi> value); @@ -778,20 +787,20 @@ class V8_EXPORT_PRIVATE CodeStubAssembler template <class... TArgs> TNode<Object> Call(TNode<Context> context, TNode<Object> callable, TNode<JSReceiver> receiver, TArgs... args) { - return UncheckedCast<Object>(CallJS( + return CallJS( CodeFactory::Call(isolate(), ConvertReceiverMode::kNotNullOrUndefined), - context, callable, receiver, args...)); + context, callable, receiver, args...); } template <class... TArgs> TNode<Object> Call(TNode<Context> context, TNode<Object> callable, TNode<Object> receiver, TArgs... args) { if (IsUndefinedConstant(receiver) || IsNullConstant(receiver)) { - return UncheckedCast<Object>(CallJS( + return CallJS( CodeFactory::Call(isolate(), ConvertReceiverMode::kNullOrUndefined), - context, callable, receiver, args...)); + context, callable, receiver, args...); } - return UncheckedCast<Object>(CallJS(CodeFactory::Call(isolate()), context, - callable, receiver, args...)); + return CallJS(CodeFactory::Call(isolate()), context, callable, receiver, + args...); } TNode<Object> CallApiCallback(TNode<Object> context, TNode<RawPtrT> callback, @@ -1128,6 +1137,10 @@ class V8_EXPORT_PRIVATE CodeStubAssembler Map::kConstructorOrBackPointerOrNativeContextOffset); } + TNode<Simd128T> LoadSimd128(TNode<IntPtrT> ptr) { + return Load<Simd128T>(ptr); + } + // Reference is the CSA-equivalent of a Torque reference value, representing // an inner pointer into a HeapObject. // @@ -1272,7 +1285,7 @@ class V8_EXPORT_PRIVATE CodeStubAssembler TNode<Int32T> instance_type, Label* bailout); // Load the identity hash of a JSRececiver. - TNode<IntPtrT> LoadJSReceiverIdentityHash(TNode<Object> receiver, + TNode<IntPtrT> LoadJSReceiverIdentityHash(TNode<JSReceiver> receiver, Label* if_no_hash = nullptr); // This is only used on a newly allocated PropertyArray which @@ -1520,6 +1533,9 @@ class V8_EXPORT_PRIVATE CodeStubAssembler void StoreHeapNumberValue(TNode<HeapNumber> object, TNode<Float64T> value); // Store a field to an object on the heap. + void StoreObjectField(TNode<HeapObject> object, int offset, TNode<Smi> value); + void StoreObjectField(TNode<HeapObject> object, TNode<IntPtrT> offset, + TNode<Smi> value); void StoreObjectField(TNode<HeapObject> object, int offset, TNode<Object> value); void StoreObjectField(TNode<HeapObject> object, TNode<IntPtrT> offset, @@ -1695,7 +1711,7 @@ class V8_EXPORT_PRIVATE CodeStubAssembler // Allocate a HeapNumber without initializing its value. TNode<HeapNumber> AllocateHeapNumber(); // Allocate a HeapNumber with a specific value. - TNode<HeapNumber> AllocateHeapNumberWithValue(SloppyTNode<Float64T> value); + TNode<HeapNumber> AllocateHeapNumberWithValue(TNode<Float64T> value); TNode<HeapNumber> AllocateHeapNumberWithValue(double value) { return AllocateHeapNumberWithValue(Float64Constant(value)); } @@ -1716,6 +1732,10 @@ class V8_EXPORT_PRIVATE CodeStubAssembler TNode<UintPtrT> LoadBigIntDigit(TNode<BigInt> bigint, TNode<IntPtrT> digit_index); + // Allocate a ByteArray with the given non-zero length. + TNode<ByteArray> AllocateNonEmptyByteArray(TNode<UintPtrT> length, + AllocationFlags flags); + // Allocate a ByteArray with the given length. TNode<ByteArray> AllocateByteArray(TNode<UintPtrT> length, AllocationFlags flags = kNone); @@ -2023,10 +2043,6 @@ class V8_EXPORT_PRIVATE CodeStubAssembler return UncheckedCast<FixedDoubleArray>(base); } - TNode<Int32T> ConvertElementsKindToInt(TNode<Int32T> elements_kind) { - return UncheckedCast<Int32T>(elements_kind); - } - template <typename T> bool ClassHasMapConstant() { return false; @@ -2242,8 +2258,8 @@ class V8_EXPORT_PRIVATE CodeStubAssembler Label* if_smi); TNode<Number> ChangeFloat32ToTagged(TNode<Float32T> value); TNode<Number> ChangeFloat64ToTagged(TNode<Float64T> value); - TNode<Number> ChangeInt32ToTagged(SloppyTNode<Int32T> value); - TNode<Number> ChangeUint32ToTagged(SloppyTNode<Uint32T> value); + TNode<Number> ChangeInt32ToTagged(TNode<Int32T> value); + TNode<Number> ChangeUint32ToTagged(TNode<Uint32T> value); TNode<Number> ChangeUintPtrToTagged(TNode<UintPtrT> value); TNode<Uint32T> ChangeNumberToUint32(TNode<Number> value); TNode<Float64T> ChangeNumberToFloat64(TNode<Number> value); @@ -2253,6 +2269,8 @@ class V8_EXPORT_PRIVATE CodeStubAssembler TNode<Float64T> ChangeTaggedToFloat64(TNode<Context> context, TNode<Object> input); + TNode<Int32T> ChangeBoolToInt32(TNode<BoolT> b); + void TaggedToNumeric(TNode<Context> context, TNode<Object> value, TVariable<Numeric>* var_numeric); void TaggedToNumericWithFeedback(TNode<Context> context, TNode<Object> value, @@ -2430,6 +2448,7 @@ class V8_EXPORT_PRIVATE CodeStubAssembler TNode<BoolT> IsStringInstanceType(TNode<Int32T> instance_type); TNode<BoolT> IsString(TNode<HeapObject> object); TNode<BoolT> IsSeqOneByteString(TNode<HeapObject> object); + TNode<BoolT> IsSwissNameDictionary(TNode<HeapObject> object); TNode<BoolT> IsSymbolInstanceType(TNode<Int32T> instance_type); TNode<BoolT> IsInternalizedStringInstanceType(TNode<Int32T> instance_type); @@ -2819,52 +2838,26 @@ class V8_EXPORT_PRIVATE CodeStubAssembler // Returns an untagged int32. template <class ContainerType> TNode<Uint32T> LoadDetailsByKeyIndex(TNode<ContainerType> container, - TNode<IntPtrT> key_index) { - static_assert(!std::is_same<ContainerType, DescriptorArray>::value, - "Use the non-templatized version for DescriptorArray"); - const int kKeyToDetailsOffset = - (ContainerType::kEntryDetailsIndex - ContainerType::kEntryKeyIndex) * - kTaggedSize; - return Unsigned(LoadAndUntagToWord32FixedArrayElement(container, key_index, - kKeyToDetailsOffset)); - } + TNode<IntPtrT> key_index); // Loads the value for the entry with the given key_index. // Returns a tagged value. template <class ContainerType> TNode<Object> LoadValueByKeyIndex(TNode<ContainerType> container, - TNode<IntPtrT> key_index) { - static_assert(!std::is_same<ContainerType, DescriptorArray>::value, - "Use the non-templatized version for DescriptorArray"); - const int kKeyToValueOffset = - (ContainerType::kEntryValueIndex - ContainerType::kEntryKeyIndex) * - kTaggedSize; - return LoadFixedArrayElement(container, key_index, kKeyToValueOffset); - } + TNode<IntPtrT> key_index); // Stores the details for the entry with the given key_index. // |details| must be a Smi. template <class ContainerType> void StoreDetailsByKeyIndex(TNode<ContainerType> container, - TNode<IntPtrT> key_index, TNode<Smi> details) { - const int kKeyToDetailsOffset = - (ContainerType::kEntryDetailsIndex - ContainerType::kEntryKeyIndex) * - kTaggedSize; - StoreFixedArrayElement(container, key_index, details, kKeyToDetailsOffset); - } + TNode<IntPtrT> key_index, TNode<Smi> details); // Stores the value for the entry with the given key_index. template <class ContainerType> void StoreValueByKeyIndex( TNode<ContainerType> container, TNode<IntPtrT> key_index, TNode<Object> value, - WriteBarrierMode write_barrier = UPDATE_WRITE_BARRIER) { - const int kKeyToValueOffset = - (ContainerType::kEntryValueIndex - ContainerType::kEntryKeyIndex) * - kTaggedSize; - StoreFixedArrayElement(container, key_index, value, write_barrier, - kKeyToValueOffset); - } + WriteBarrierMode write_barrier = UPDATE_WRITE_BARRIER); // Calculate a valid size for the a hash table. TNode<IntPtrT> HashTableComputeCapacity(TNode<IntPtrT> at_least_space_for); @@ -2880,12 +2873,18 @@ class V8_EXPORT_PRIVATE CodeStubAssembler template <class Dictionary> void SetNumberOfElements(TNode<Dictionary> dictionary, TNode<Smi> num_elements_smi) { + // Not supposed to be used for SwissNameDictionary. + STATIC_ASSERT(!(std::is_same<Dictionary, SwissNameDictionary>::value)); + StoreFixedArrayElement(dictionary, Dictionary::kNumberOfElementsIndex, num_elements_smi, SKIP_WRITE_BARRIER); } template <class Dictionary> TNode<Smi> GetNumberOfDeletedElements(TNode<Dictionary> dictionary) { + // Not supposed to be used for SwissNameDictionary. + STATIC_ASSERT(!(std::is_same<Dictionary, SwissNameDictionary>::value)); + return CAST(LoadFixedArrayElement( dictionary, Dictionary::kNumberOfDeletedElementsIndex)); } @@ -2893,6 +2892,9 @@ class V8_EXPORT_PRIVATE CodeStubAssembler template <class Dictionary> void SetNumberOfDeletedElements(TNode<Dictionary> dictionary, TNode<Smi> num_deleted_smi) { + // Not supposed to be used for SwissNameDictionary. + STATIC_ASSERT(!(std::is_same<Dictionary, SwissNameDictionary>::value)); + StoreFixedArrayElement(dictionary, Dictionary::kNumberOfDeletedElementsIndex, num_deleted_smi, SKIP_WRITE_BARRIER); @@ -2900,6 +2902,9 @@ class V8_EXPORT_PRIVATE CodeStubAssembler template <class Dictionary> TNode<Smi> GetCapacity(TNode<Dictionary> dictionary) { + // Not supposed to be used for SwissNameDictionary. + STATIC_ASSERT(!(std::is_same<Dictionary, SwissNameDictionary>::value)); + return CAST( UnsafeLoadFixedArrayElement(dictionary, Dictionary::kCapacityIndex)); } @@ -3040,10 +3045,11 @@ class V8_EXPORT_PRIVATE CodeStubAssembler TNode<IntPtrT> name_index, TNode<Uint32T>, TVariable<Object>* var_value); - void LoadPropertyFromNameDictionary(TNode<NameDictionary> dictionary, - TNode<IntPtrT> name_index, - TVariable<Uint32T>* var_details, - TVariable<Object>* var_value); + template <typename Dictionary> + void LoadPropertyFromDictionary(TNode<Dictionary> dictionary, + TNode<IntPtrT> name_index, + TVariable<Uint32T>* var_details, + TVariable<Object>* var_value); void LoadPropertyFromGlobalDictionary(TNode<GlobalDictionary> dictionary, TNode<IntPtrT> name_index, TVariable<Uint32T>* var_details, @@ -3179,7 +3185,7 @@ class V8_EXPORT_PRIVATE CodeStubAssembler // Report that there was a feedback update, performing any tasks that should // be done after a feedback update. void ReportFeedbackUpdate(TNode<FeedbackVector> feedback_vector, - SloppyTNode<UintPtrT> slot_id, const char* reason); + TNode<UintPtrT> slot_id, const char* reason); // Combine the new feedback with the existing_feedback. Do nothing if // existing_feedback is nullptr. @@ -3203,10 +3209,6 @@ class V8_EXPORT_PRIVATE CodeStubAssembler TNode<Uint8T> Int32ToUint8Clamped(TNode<Int32T> int32_value); TNode<Uint8T> Float64ToUint8Clamped(TNode<Float64T> float64_value); - Node* PrepareValueForWriteToTypedArray(TNode<Object> input, - ElementsKind elements_kind, - TNode<Context> context); - template <typename T> TNode<T> PrepareValueForWriteToTypedArray(TNode<Object> input, ElementsKind elements_kind, @@ -3216,9 +3218,9 @@ class V8_EXPORT_PRIVATE CodeStubAssembler // TODO(turbofan): For BIGINT64_ELEMENTS and BIGUINT64_ELEMENTS // we pass {value} as BigInt object instead of int64_t. We should // teach TurboFan to handle int64_t on 32-bit platforms eventually. - template <typename TIndex> + template <typename TIndex, typename TValue> void StoreElement(TNode<RawPtrT> elements, ElementsKind kind, - TNode<TIndex> index, Node* value); + TNode<TIndex> index, TNode<TValue> value); // Implements the BigInt part of // https://tc39.github.io/proposal-bigint/#sec-numbertorawbytes, @@ -3487,52 +3489,25 @@ class V8_EXPORT_PRIVATE CodeStubAssembler TNode<Code> LoadBuiltin(TNode<Smi> builtin_id); // Figure out the SFI's code object using its data field. + // If |data_type_out| is provided, the instance type of the function data will + // be stored in it. In case the code object is a builtin (data is a Smi), + // data_type_out will be set to 0. // If |if_compile_lazy| is provided then the execution will go to the given // label in case of an CompileLazy code object. - TNode<Code> GetSharedFunctionInfoCode(TNode<SharedFunctionInfo> shared_info, - Label* if_compile_lazy = nullptr); + TNode<Code> GetSharedFunctionInfoCode( + TNode<SharedFunctionInfo> shared_info, + TVariable<Uint16T>* data_type_out = nullptr, + Label* if_compile_lazy = nullptr); TNode<JSFunction> AllocateFunctionWithMapAndContext( TNode<Map> map, TNode<SharedFunctionInfo> shared_info, TNode<Context> context); // Promise helpers - TNode<Uint32T> PromiseHookFlags(); + TNode<BoolT> IsPromiseHookEnabled(); TNode<BoolT> HasAsyncEventDelegate(); - TNode<BoolT> IsContextPromiseHookEnabled(TNode<Uint32T> flags); - TNode<BoolT> IsContextPromiseHookEnabled() { - return IsContextPromiseHookEnabled(PromiseHookFlags()); - } - TNode<BoolT> IsAnyPromiseHookEnabled(TNode<Uint32T> flags); - TNode<BoolT> IsAnyPromiseHookEnabled() { - return IsAnyPromiseHookEnabled(PromiseHookFlags()); - } - TNode<BoolT> IsIsolatePromiseHookEnabledOrHasAsyncEventDelegate( - TNode<Uint32T> flags); - TNode<BoolT> IsIsolatePromiseHookEnabledOrHasAsyncEventDelegate() { - return IsIsolatePromiseHookEnabledOrHasAsyncEventDelegate( - PromiseHookFlags()); - } - TNode<BoolT> - IsIsolatePromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate( - TNode<Uint32T> flags); - TNode<BoolT> - IsIsolatePromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate() { - return IsIsolatePromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate( - PromiseHookFlags()); - } - TNode<BoolT> IsAnyPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate( - TNode<Uint32T> flags); - TNode<BoolT> - IsAnyPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate() { - return IsAnyPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate( - PromiseHookFlags()); - } - - TNode<BoolT> NeedsAnyPromiseHooks(TNode<Uint32T> flags); - TNode<BoolT> NeedsAnyPromiseHooks() { - return NeedsAnyPromiseHooks(PromiseHookFlags()); - } + TNode<BoolT> IsPromiseHookEnabledOrHasAsyncEventDelegate(); + TNode<BoolT> IsPromiseHookEnabledOrDebugIsActiveOrHasAsyncEventDelegate(); // for..in helpers void CheckPrototypeEnumCache(TNode<JSReceiver> receiver, @@ -3579,6 +3554,8 @@ class V8_EXPORT_PRIVATE CodeStubAssembler bool ConstexprInt32GreaterThanEqual(int32_t a, int32_t b) { return a >= b; } uint32_t ConstexprUint32Add(uint32_t a, uint32_t b) { return a + b; } int32_t ConstexprUint32Sub(uint32_t a, uint32_t b) { return a - b; } + int32_t ConstexprInt32Sub(int32_t a, int32_t b) { return a - b; } + int32_t ConstexprInt32Add(int32_t a, int32_t b) { return a + b; } int31_t ConstexprInt31Add(int31_t a, int31_t b) { int32_t val; CHECK(!base::bits::SignedAddOverflow32(a, b, &val)); @@ -3733,6 +3710,88 @@ class V8_EXPORT_PRIVATE CodeStubAssembler return IntPtrConstant(FeedbackIterator::kHandlerOffset); } + TNode<SwissNameDictionary> AllocateSwissNameDictionary( + TNode<IntPtrT> at_least_space_for); + TNode<SwissNameDictionary> AllocateSwissNameDictionary( + int at_least_space_for); + + TNode<SwissNameDictionary> AllocateSwissNameDictionaryWithCapacity( + TNode<IntPtrT> capacity); + + // MT stands for "minus tag". + TNode<IntPtrT> SwissNameDictionaryOffsetIntoDataTableMT( + TNode<SwissNameDictionary> dict, TNode<IntPtrT> index, int field_index); + + // MT stands for "minus tag". + TNode<IntPtrT> SwissNameDictionaryOffsetIntoPropertyDetailsTableMT( + TNode<SwissNameDictionary> dict, TNode<IntPtrT> capacity, + TNode<IntPtrT> index); + + TNode<IntPtrT> LoadSwissNameDictionaryNumberOfElements( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity); + + TNode<IntPtrT> LoadSwissNameDictionaryNumberOfDeletedElements( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity); + + // Specialized operation to be used when adding entries: + // If used capacity (= number of present + deleted elements) is less than + // |max_usable|, increment the number of present entries and return the used + // capacity value (prior to the incrementation). Otherwise, goto |bailout|. + TNode<Uint32T> SwissNameDictionaryIncreaseElementCountOrBailout( + TNode<ByteArray> meta_table, TNode<IntPtrT> capacity, + TNode<Uint32T> max_usable_capacity, Label* bailout); + + // Specialized operation to be used when deleting entries: Decreases the + // number of present entries and increases the number of deleted ones. Returns + // new (= decremented) number of present entries. + TNode<Uint32T> SwissNameDictionaryUpdateCountsForDeletion( + TNode<ByteArray> meta_table, TNode<IntPtrT> capacity); + + void StoreSwissNameDictionaryCapacity(TNode<SwissNameDictionary> table, + TNode<Int32T> capacity); + + void StoreSwissNameDictionaryEnumToEntryMapping( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity, + TNode<IntPtrT> enum_index, TNode<Int32T> entry); + + TNode<Name> LoadSwissNameDictionaryKey(TNode<SwissNameDictionary> dict, + TNode<IntPtrT> entry); + + void StoreSwissNameDictionaryKeyAndValue(TNode<SwissNameDictionary> dict, + TNode<IntPtrT> entry, + TNode<Object> key, + TNode<Object> value); + + // Equivalent to SwissNameDictionary::SetCtrl, therefore preserves the copy of + // the first group at the end of the control table. + void SwissNameDictionarySetCtrl(TNode<SwissNameDictionary> table, + TNode<IntPtrT> capacity, TNode<IntPtrT> entry, + TNode<Uint8T> ctrl); + + TNode<Uint64T> LoadSwissNameDictionaryCtrlTableGroup(TNode<IntPtrT> address); + + TNode<Uint8T> LoadSwissNameDictionaryPropertyDetails( + TNode<SwissNameDictionary> table, TNode<IntPtrT> capacity, + TNode<IntPtrT> entry); + + void StoreSwissNameDictionaryPropertyDetails(TNode<SwissNameDictionary> table, + TNode<IntPtrT> capacity, + TNode<IntPtrT> entry, + TNode<Uint8T> details); + + TNode<SwissNameDictionary> CopySwissNameDictionary( + TNode<SwissNameDictionary> original); + + void SwissNameDictionaryFindEntry(TNode<SwissNameDictionary> table, + TNode<Name> key, Label* found, + TVariable<IntPtrT>* var_found_entry, + Label* not_found); + + void SwissNameDictionaryAdd(TNode<SwissNameDictionary> table, TNode<Name> key, + TNode<Object> value, + TNode<Uint8T> property_details, + Label* needs_resize); + private: friend class CodeStubArguments; @@ -3831,15 +3890,23 @@ class V8_EXPORT_PRIVATE CodeStubAssembler TNode<Object> value, WriteBarrierMode barrier_mode = UPDATE_WRITE_BARRIER, int additional_offset = 0); + template <typename TIndex> + void StoreElementTypedArrayBigInt(TNode<RawPtrT> elements, ElementsKind kind, + TNode<TIndex> index, TNode<BigInt> value); + + template <typename TIndex> + void StoreElementTypedArrayWord32(TNode<RawPtrT> elements, ElementsKind kind, + TNode<TIndex> index, TNode<Word32T> value); + // Store value to an elements array with given elements kind. // TODO(turbofan): For BIGINT64_ELEMENTS and BIGUINT64_ELEMENTS // we pass {value} as BigInt object instead of int64_t. We should // teach TurboFan to handle int64_t on 32-bit platforms eventually. // TODO(solanes): This method can go away and simplify into only one version // of StoreElement once we have "if constexpr" available to use. - template <typename TArray, typename TIndex> + template <typename TArray, typename TIndex, typename TValue> void StoreElementTypedArray(TNode<TArray> elements, ElementsKind kind, - TNode<TIndex> index, Node* value); + TNode<TIndex> index, TNode<TValue> value); template <typename TIndex> void StoreElement(TNode<FixedArrayBase> elements, ElementsKind kind, @@ -3855,12 +3922,23 @@ class V8_EXPORT_PRIVATE CodeStubAssembler void TryPlainPrimitiveNonNumberToNumber(TNode<HeapObject> input, TVariable<Number>* var_result, Label* if_bailout); + + template <typename TValue> + void EmitElementStoreTypedArray(TNode<JSTypedArray> typed_array, + TNode<IntPtrT> key, TNode<Object> value, + ElementsKind elements_kind, + KeyedAccessStoreMode store_mode, + Label* bailout, TNode<Context> context, + TVariable<Object>* maybe_converted_value); + + template <typename TValue> + void EmitElementStoreTypedArrayUpdateValue( + TNode<Object> value, ElementsKind elements_kind, + TNode<TValue> converted_value, TVariable<Object>* maybe_converted_value); }; class V8_EXPORT_PRIVATE CodeStubArguments { public: - using Node = compiler::Node; - // |argc| specifies the number of arguments passed to the builtin excluding // the receiver. The arguments include the receiver. CodeStubArguments(CodeStubAssembler* assembler, TNode<IntPtrT> argc) diff --git a/deps/v8/src/codegen/compilation-cache.cc b/deps/v8/src/codegen/compilation-cache.cc index 826b53293a..3941e56e6a 100644 --- a/deps/v8/src/codegen/compilation-cache.cc +++ b/deps/v8/src/codegen/compilation-cache.cc @@ -145,7 +145,7 @@ bool CompilationCacheScript::HasOrigin(Handle<SharedFunctionInfo> function_info, MaybeHandle<SharedFunctionInfo> CompilationCacheScript::Lookup( Handle<String> source, MaybeHandle<Object> name, int line_offset, int column_offset, ScriptOriginOptions resource_options, - Handle<Context> native_context, LanguageMode language_mode) { + LanguageMode language_mode) { MaybeHandle<SharedFunctionInfo> result; // Probe the script generation tables. Make sure not to leak handles @@ -156,7 +156,7 @@ MaybeHandle<SharedFunctionInfo> CompilationCacheScript::Lookup( DCHECK_EQ(generations(), 1); Handle<CompilationCacheTable> table = GetTable(generation); MaybeHandle<SharedFunctionInfo> probe = CompilationCacheTable::LookupScript( - table, source, native_context, language_mode); + table, source, language_mode, isolate()); Handle<SharedFunctionInfo> function_info; if (probe.ToHandle(&function_info)) { // Break when we've found a suitable shared function info that @@ -188,13 +188,12 @@ MaybeHandle<SharedFunctionInfo> CompilationCacheScript::Lookup( } void CompilationCacheScript::Put(Handle<String> source, - Handle<Context> native_context, LanguageMode language_mode, Handle<SharedFunctionInfo> function_info) { HandleScope scope(isolate()); Handle<CompilationCacheTable> table = GetFirstTable(); - SetFirstTable(CompilationCacheTable::PutScript(table, source, native_context, - language_mode, function_info)); + SetFirstTable(CompilationCacheTable::PutScript(table, source, language_mode, + function_info, isolate())); } InfoCellPair CompilationCacheEval::Lookup(Handle<String> source, @@ -331,11 +330,11 @@ void CompilationCache::Remove(Handle<SharedFunctionInfo> function_info) { MaybeHandle<SharedFunctionInfo> CompilationCache::LookupScript( Handle<String> source, MaybeHandle<Object> name, int line_offset, int column_offset, ScriptOriginOptions resource_options, - Handle<Context> native_context, LanguageMode language_mode) { + LanguageMode language_mode) { if (!IsEnabledScriptAndEval()) return MaybeHandle<SharedFunctionInfo>(); return script_.Lookup(source, name, line_offset, column_offset, - resource_options, native_context, language_mode); + resource_options, language_mode); } InfoCellPair CompilationCache::LookupEval(Handle<String> source, @@ -378,13 +377,12 @@ MaybeHandle<Code> CompilationCache::LookupCode(Handle<SharedFunctionInfo> sfi) { } void CompilationCache::PutScript(Handle<String> source, - Handle<Context> native_context, LanguageMode language_mode, Handle<SharedFunctionInfo> function_info) { if (!IsEnabledScriptAndEval()) return; LOG(isolate(), CompilationCacheEvent("put", "script", *function_info)); - script_.Put(source, native_context, language_mode, function_info); + script_.Put(source, language_mode, function_info); } void CompilationCache::PutEval(Handle<String> source, diff --git a/deps/v8/src/codegen/compilation-cache.h b/deps/v8/src/codegen/compilation-cache.h index 58e4e0f75f..0ed13e53b6 100644 --- a/deps/v8/src/codegen/compilation-cache.h +++ b/deps/v8/src/codegen/compilation-cache.h @@ -85,11 +85,9 @@ class CompilationCacheScript : public CompilationSubCache { MaybeHandle<Object> name, int line_offset, int column_offset, ScriptOriginOptions resource_options, - Handle<Context> native_context, LanguageMode language_mode); - void Put(Handle<String> source, Handle<Context> context, - LanguageMode language_mode, + void Put(Handle<String> source, LanguageMode language_mode, Handle<SharedFunctionInfo> function_info); void Age() override; @@ -193,7 +191,7 @@ class V8_EXPORT_PRIVATE CompilationCache { MaybeHandle<SharedFunctionInfo> LookupScript( Handle<String> source, MaybeHandle<Object> name, int line_offset, int column_offset, ScriptOriginOptions resource_options, - Handle<Context> native_context, LanguageMode language_mode); + LanguageMode language_mode); // Finds the shared function info for a source string for eval in a // given context. Returns an empty handle if the cache doesn't @@ -212,8 +210,7 @@ class V8_EXPORT_PRIVATE CompilationCache { // Associate the (source, kind) pair to the shared function // info. This may overwrite an existing mapping. - void PutScript(Handle<String> source, Handle<Context> native_context, - LanguageMode language_mode, + void PutScript(Handle<String> source, LanguageMode language_mode, Handle<SharedFunctionInfo> function_info); // Associate the (source, context->closure()->shared(), kind) triple diff --git a/deps/v8/src/codegen/compiler.cc b/deps/v8/src/codegen/compiler.cc index 66336ca32c..e46639d90a 100644 --- a/deps/v8/src/codegen/compiler.cc +++ b/deps/v8/src/codegen/compiler.cc @@ -288,7 +288,6 @@ ScriptOriginOptions OriginOptionsForEval(Object script) { // Implementation of UnoptimizedCompilationJob CompilationJob::Status UnoptimizedCompilationJob::ExecuteJob() { - DisallowHeapAccess no_heap_access; // Delegate to the underlying implementation. DCHECK_EQ(state(), State::kReadyToExecute); ScopedTimer t(&time_taken_to_execute_); @@ -319,12 +318,15 @@ namespace { void RecordUnoptimizedCompilationStats(Isolate* isolate, Handle<SharedFunctionInfo> shared_info) { - int code_size; - if (shared_info->HasBytecodeArray()) { - code_size = shared_info->GetBytecodeArray(isolate).SizeIncludingMetadata(); - } else { - code_size = shared_info->asm_wasm_data().Size(); - } +#if V8_ENABLE_WEBASSEMBLY + int code_size = + shared_info->HasBytecodeArray() + ? shared_info->GetBytecodeArray(isolate).SizeIncludingMetadata() + : shared_info->asm_wasm_data().Size(); +#else + int code_size = + shared_info->GetBytecodeArray(isolate).SizeIncludingMetadata(); +#endif // V8_ENABLE_WEBASSEMBLY Counters* counters = isolate->counters(); // TODO(4280): Rename counters from "baseline" to "unoptimized" eventually. @@ -344,9 +346,13 @@ void RecordUnoptimizedFunctionCompilation( abstract_code = handle(AbstractCode::cast(shared->GetBytecodeArray(isolate)), isolate); } else { +#if V8_ENABLE_WEBASSEMBLY DCHECK(shared->HasAsmWasmData()); abstract_code = Handle<AbstractCode>::cast(BUILTIN_CODE(isolate, InstantiateAsmJs)); +#else + UNREACHABLE(); +#endif // V8_ENABLE_WEBASSEMBLY } double time_taken_ms = time_taken_to_execute.InMillisecondsF() + @@ -516,8 +522,9 @@ bool UseAsmWasm(FunctionLiteral* literal, bool asm_wasm_broken) { } #endif -void InstallInterpreterTrampolineCopy(Isolate* isolate, - Handle<SharedFunctionInfo> shared_info) { +void InstallInterpreterTrampolineCopy( + Isolate* isolate, Handle<SharedFunctionInfo> shared_info, + CodeEventListener::LogEventsAndTags log_tag) { DCHECK(FLAG_interpreted_frames_native_stack); if (!shared_info->function_data(kAcquireLoad).IsBytecodeArray()) { DCHECK(!shared_info->HasBytecodeArray()); @@ -548,8 +555,6 @@ void InstallInterpreterTrampolineCopy(Isolate* isolate, handle(script->name().IsString() ? String::cast(script->name()) : ReadOnlyRoots(isolate).empty_string(), isolate); - CodeEventListener::LogEventsAndTags log_tag = Logger::ToNativeByScript( - CodeEventListener::INTERPRETED_FUNCTION_TAG, *script); PROFILE(isolate, CodeCreateEvent(log_tag, abstract_code, shared_info, script_name, line_num, column_num)); } @@ -563,11 +568,13 @@ void InstallUnoptimizedCode(UnoptimizedCompilationInfo* compilation_info, DCHECK(!compilation_info->has_asm_wasm_data()); DCHECK(!shared_info->HasFeedbackMetadata()); +#if V8_ENABLE_WEBASSEMBLY // If the function failed asm-wasm compilation, mark asm_wasm as broken // to ensure we don't try to compile as asm-wasm. if (compilation_info->literal()->scope()->IsAsmModule()) { shared_info->set_is_asm_wasm_broken(true); } +#endif // V8_ENABLE_WEBASSEMBLY shared_info->set_bytecode_array(*compilation_info->bytecode_array()); @@ -575,29 +582,24 @@ void InstallUnoptimizedCode(UnoptimizedCompilationInfo* compilation_info, isolate, compilation_info->feedback_vector_spec()); shared_info->set_feedback_metadata(*feedback_metadata); } else { +#if V8_ENABLE_WEBASSEMBLY DCHECK(compilation_info->has_asm_wasm_data()); // We should only have asm/wasm data when finalizing on the main thread. DCHECK((std::is_same<LocalIsolate, Isolate>::value)); shared_info->set_asm_wasm_data(*compilation_info->asm_wasm_data()); shared_info->set_feedback_metadata( ReadOnlyRoots(isolate).empty_feedback_metadata()); +#else + UNREACHABLE(); +#endif // V8_ENABLE_WEBASSEMBLY } } void LogUnoptimizedCompilation(Isolate* isolate, Handle<SharedFunctionInfo> shared_info, - UnoptimizedCompileFlags flags, + CodeEventListener::LogEventsAndTags log_tag, base::TimeDelta time_taken_to_execute, base::TimeDelta time_taken_to_finalize) { - CodeEventListener::LogEventsAndTags log_tag; - if (flags.is_toplevel()) { - log_tag = flags.is_eval() ? CodeEventListener::EVAL_TAG - : CodeEventListener::SCRIPT_TAG; - } else { - log_tag = flags.is_lazy_compile() ? CodeEventListener::LAZY_COMPILE_TAG - : CodeEventListener::FUNCTION_TAG; - } - RecordUnoptimizedFunctionCompilation(isolate, log_tag, shared_info, time_taken_to_execute, time_taken_to_finalize); @@ -684,7 +686,11 @@ bool CompileSharedWithBaseline(Isolate* isolate, base::TimeDelta time_taken; { ScopedTimer timer(&time_taken); - code = GenerateBaselineCode(isolate, shared); + if (!GenerateBaselineCode(isolate, shared).ToHandle(&code)) { + // TODO(leszeks): This can only fail because of an OOM. Do we want to + // report these somehow, or silently ignore them? + return false; + } Handle<HeapObject> function_data = handle(HeapObject::cast(shared->function_data(kAcquireLoad)), isolate); @@ -1341,19 +1347,24 @@ void FinalizeUnoptimizedCompilation( if (need_source_positions) { SharedFunctionInfo::EnsureSourcePositionsAvailable(isolate, shared_info); } - if (FLAG_interpreted_frames_native_stack) { - InstallInterpreterTrampolineCopy(isolate, shared_info); + CodeEventListener::LogEventsAndTags log_tag; + if (shared_info->is_toplevel()) { + log_tag = flags.is_eval() ? CodeEventListener::EVAL_TAG + : CodeEventListener::SCRIPT_TAG; + } else { + log_tag = flags.is_lazy_compile() ? CodeEventListener::LAZY_COMPILE_TAG + : CodeEventListener::FUNCTION_TAG; } - if (FLAG_always_sparkplug) { - CompileSharedWithBaseline(isolate, shared_info, Compiler::KEEP_EXCEPTION, - &is_compiled_scope); + log_tag = Logger::ToNativeByScript(log_tag, *script); + if (FLAG_interpreted_frames_native_stack) { + InstallInterpreterTrampolineCopy(isolate, shared_info, log_tag); } Handle<CoverageInfo> coverage_info; if (finalize_data.coverage_info().ToHandle(&coverage_info)) { isolate->debug()->InstallCoverageInfo(shared_info, coverage_info); } - LogUnoptimizedCompilation(isolate, shared_info, flags, + LogUnoptimizedCompilation(isolate, shared_info, log_tag, finalize_data.time_taken_to_execute(), finalize_data.time_taken_to_finalize()); } @@ -1394,6 +1405,19 @@ void FinalizeUnoptimizedScriptCompilation( } } +void CompileAllWithBaseline(Isolate* isolate, + const FinalizeUnoptimizedCompilationDataList& + finalize_unoptimized_compilation_data_list) { + for (const auto& finalize_data : finalize_unoptimized_compilation_data_list) { + Handle<SharedFunctionInfo> shared_info = finalize_data.function_handle(); + IsCompiledScope is_compiled_scope(*shared_info, isolate); + if (!is_compiled_scope.is_compiled()) continue; + if (!CanCompileWithBaseline(isolate, shared_info)) continue; + CompileSharedWithBaseline(isolate, shared_info, Compiler::CLEAR_EXCEPTION, + &is_compiled_scope); + } +} + // Create shared function info for top level and shared function infos array for // inner functions. template <typename LocalIsolate> @@ -1462,6 +1486,11 @@ MaybeHandle<SharedFunctionInfo> CompileToplevel( FinalizeUnoptimizedScriptCompilation( isolate, script, parse_info->flags(), parse_info->state(), finalize_unoptimized_compilation_data_list); + + if (FLAG_always_sparkplug) { + CompileAllWithBaseline(isolate, finalize_unoptimized_compilation_data_list); + } + return shared_info; } @@ -1919,6 +1948,10 @@ bool Compiler::Compile(Isolate* isolate, Handle<SharedFunctionInfo> shared_info, FinalizeUnoptimizedCompilation(isolate, script, flags, &compile_state, finalize_unoptimized_compilation_data_list); + if (FLAG_always_sparkplug) { + CompileAllWithBaseline(isolate, finalize_unoptimized_compilation_data_list); + } + DCHECK(!isolate->has_pending_exception()); DCHECK(is_compiled_scope->is_compiled()); return true; @@ -1958,13 +1991,12 @@ bool Compiler::Compile(Isolate* isolate, Handle<JSFunction> function, // immediately after a flush would be better. JSFunction::InitializeFeedbackCell(function, is_compiled_scope, true); - // If --always-sparkplug is enabled, make sure we have baseline code. - if (FLAG_always_sparkplug && CanCompileWithBaseline(isolate, shared_info)) { - DCHECK(shared_info->HasBaselineData()); - } - // Optimize now if --always-opt is enabled. +#if V8_ENABLE_WEBASSEMBLY if (FLAG_always_opt && !function->shared().HasAsmWasmData()) { +#else + if (FLAG_always_opt) { +#endif // V8_ENABLE_WEBASSEMBLY CompilerTracer::TraceOptimizeForAlwaysOpt(isolate, function, CodeKindForTopTier()); @@ -2769,7 +2801,11 @@ MaybeHandle<SharedFunctionInfo> CompileScriptOnBothBackgroundAndMainThread( } // Join with background thread and finalize compilation. - background_compile_thread.Join(); + { + ParkedScope scope(isolate->main_thread_local_isolate()); + background_compile_thread.Join(); + } + MaybeHandle<SharedFunctionInfo> maybe_result = Compiler::GetSharedFunctionInfoForStreamedScript( isolate, source, script_details, origin_options, @@ -2838,8 +2874,7 @@ MaybeHandle<SharedFunctionInfo> Compiler::GetSharedFunctionInfoForScript( // First check per-isolate compilation cache. maybe_result = compilation_cache->LookupScript( source, script_details.name_obj, script_details.line_offset, - script_details.column_offset, origin_options, isolate->native_context(), - language_mode); + script_details.column_offset, origin_options, language_mode); if (!maybe_result.is_null()) { compile_timer.set_hit_isolate_cache(); } else if (can_consume_code_cache) { @@ -2858,8 +2893,7 @@ MaybeHandle<SharedFunctionInfo> Compiler::GetSharedFunctionInfoForScript( // Promote to per-isolate compilation cache. is_compiled_scope = inner_result->is_compiled_scope(isolate); DCHECK(is_compiled_scope.is_compiled()); - compilation_cache->PutScript(source, isolate->native_context(), - language_mode, inner_result); + compilation_cache->PutScript(source, language_mode, inner_result); Handle<Script> script(Script::cast(inner_result->script()), isolate); maybe_result = inner_result; } else { @@ -2897,8 +2931,7 @@ MaybeHandle<SharedFunctionInfo> Compiler::GetSharedFunctionInfoForScript( Handle<SharedFunctionInfo> result; if (use_compilation_cache && maybe_result.ToHandle(&result)) { DCHECK(is_compiled_scope.is_compiled()); - compilation_cache->PutScript(source, isolate->native_context(), - language_mode, result); + compilation_cache->PutScript(source, language_mode, result); } else if (maybe_result.is_null() && natives != EXTENSION_CODE) { isolate->ReportPendingMessages(); } @@ -3028,8 +3061,7 @@ Compiler::GetSharedFunctionInfoForStreamedScript( "V8.StreamingFinalization.CheckCache"); maybe_result = compilation_cache->LookupScript( source, script_details.name_obj, script_details.line_offset, - script_details.column_offset, origin_options, isolate->native_context(), - task->language_mode()); + script_details.column_offset, origin_options, task->language_mode()); if (!maybe_result.is_null()) { compile_timer.set_hit_isolate_cache(); } @@ -3038,6 +3070,7 @@ Compiler::GetSharedFunctionInfoForStreamedScript( if (maybe_result.is_null()) { // No cache entry found, finalize compilation of the script and add it to // the isolate cache. + DCHECK_EQ(task->flags().is_module(), origin_options.IsModule()); Handle<Script> script; if (FLAG_finalize_streaming_on_background && !origin_options.IsModule()) { @@ -3068,8 +3101,8 @@ Compiler::GetSharedFunctionInfoForStreamedScript( isolate->heap()->SetRootScriptList(*scripts); } else { ParseInfo* parse_info = task->info(); - DCHECK(parse_info->flags().is_toplevel()); DCHECK_EQ(parse_info->flags().is_module(), origin_options.IsModule()); + DCHECK(parse_info->flags().is_toplevel()); script = parse_info->CreateScript(isolate, source, kNullMaybeHandle, origin_options); @@ -3119,8 +3152,7 @@ Compiler::GetSharedFunctionInfoForStreamedScript( // Add compiled code to the isolate cache. TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"), "V8.StreamingFinalization.AddToCache"); - compilation_cache->PutScript(source, isolate->native_context(), - task->language_mode(), result); + compilation_cache->PutScript(source, task->language_mode(), result); } } diff --git a/deps/v8/src/codegen/cpu-features.h b/deps/v8/src/codegen/cpu-features.h index 7cb45c7785..b9a450ea3a 100644 --- a/deps/v8/src/codegen/cpu-features.h +++ b/deps/v8/src/codegen/cpu-features.h @@ -58,6 +58,7 @@ enum CpuFeature { ISELECT, VSX, MODULO, + SIMD, #elif V8_TARGET_ARCH_S390X FPU, @@ -108,9 +109,9 @@ class V8_EXPORT_PRIVATE CpuFeatures : public AllStatic { return (supported_ & (1u << f)) != 0; } - static inline bool SupportsOptimizer(); + static bool SupportsWasmSimd128(); - static inline bool SupportsWasmSimd128(); + static inline bool SupportsOptimizer(); static inline unsigned icache_line_size() { DCHECK_NE(icache_line_size_, 0); diff --git a/deps/v8/src/codegen/external-reference.cc b/deps/v8/src/codegen/external-reference.cc index 88ce90f4fd..454b04e893 100644 --- a/deps/v8/src/codegen/external-reference.cc +++ b/deps/v8/src/codegen/external-reference.cc @@ -30,7 +30,10 @@ #include "src/regexp/regexp-macro-assembler-arch.h" #include "src/regexp/regexp-stack.h" #include "src/strings/string-search.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-external-refs.h" +#endif // V8_ENABLE_WEBASSEMBLY #ifdef V8_INTL_SUPPORT #include "src/base/platform/wrappers.h" @@ -335,53 +338,66 @@ FUNCTION_REFERENCE(new_deoptimizer_function, Deoptimizer::New) FUNCTION_REFERENCE(compute_output_frames_function, Deoptimizer::ComputeOutputFrames) -FUNCTION_REFERENCE(wasm_f32_trunc, wasm::f32_trunc_wrapper) -FUNCTION_REFERENCE(wasm_f32_floor, wasm::f32_floor_wrapper) -FUNCTION_REFERENCE(wasm_f32_ceil, wasm::f32_ceil_wrapper) -FUNCTION_REFERENCE(wasm_f32_nearest_int, wasm::f32_nearest_int_wrapper) -FUNCTION_REFERENCE(wasm_f64_trunc, wasm::f64_trunc_wrapper) -FUNCTION_REFERENCE(wasm_f64_floor, wasm::f64_floor_wrapper) -FUNCTION_REFERENCE(wasm_f64_ceil, wasm::f64_ceil_wrapper) -FUNCTION_REFERENCE(wasm_f64_nearest_int, wasm::f64_nearest_int_wrapper) -FUNCTION_REFERENCE(wasm_int64_to_float32, wasm::int64_to_float32_wrapper) -FUNCTION_REFERENCE(wasm_uint64_to_float32, wasm::uint64_to_float32_wrapper) -FUNCTION_REFERENCE(wasm_int64_to_float64, wasm::int64_to_float64_wrapper) -FUNCTION_REFERENCE(wasm_uint64_to_float64, wasm::uint64_to_float64_wrapper) -FUNCTION_REFERENCE(wasm_float32_to_int64, wasm::float32_to_int64_wrapper) -FUNCTION_REFERENCE(wasm_float32_to_uint64, wasm::float32_to_uint64_wrapper) -FUNCTION_REFERENCE(wasm_float64_to_int64, wasm::float64_to_int64_wrapper) -FUNCTION_REFERENCE(wasm_float64_to_uint64, wasm::float64_to_uint64_wrapper) -FUNCTION_REFERENCE(wasm_float32_to_int64_sat, - wasm::float32_to_int64_sat_wrapper) -FUNCTION_REFERENCE(wasm_float32_to_uint64_sat, - wasm::float32_to_uint64_sat_wrapper) -FUNCTION_REFERENCE(wasm_float64_to_int64_sat, - wasm::float64_to_int64_sat_wrapper) -FUNCTION_REFERENCE(wasm_float64_to_uint64_sat, - wasm::float64_to_uint64_sat_wrapper) -FUNCTION_REFERENCE(wasm_int64_div, wasm::int64_div_wrapper) -FUNCTION_REFERENCE(wasm_int64_mod, wasm::int64_mod_wrapper) -FUNCTION_REFERENCE(wasm_uint64_div, wasm::uint64_div_wrapper) -FUNCTION_REFERENCE(wasm_uint64_mod, wasm::uint64_mod_wrapper) -FUNCTION_REFERENCE(wasm_word32_ctz, wasm::word32_ctz_wrapper) -FUNCTION_REFERENCE(wasm_word64_ctz, wasm::word64_ctz_wrapper) -FUNCTION_REFERENCE(wasm_word32_popcnt, wasm::word32_popcnt_wrapper) -FUNCTION_REFERENCE(wasm_word64_popcnt, wasm::word64_popcnt_wrapper) -FUNCTION_REFERENCE(wasm_word32_rol, wasm::word32_rol_wrapper) -FUNCTION_REFERENCE(wasm_word32_ror, wasm::word32_ror_wrapper) -FUNCTION_REFERENCE(wasm_word64_rol, wasm::word64_rol_wrapper) -FUNCTION_REFERENCE(wasm_word64_ror, wasm::word64_ror_wrapper) -FUNCTION_REFERENCE(wasm_f64x2_ceil, wasm::f64x2_ceil_wrapper) -FUNCTION_REFERENCE(wasm_f64x2_floor, wasm::f64x2_floor_wrapper) -FUNCTION_REFERENCE(wasm_f64x2_trunc, wasm::f64x2_trunc_wrapper) -FUNCTION_REFERENCE(wasm_f64x2_nearest_int, wasm::f64x2_nearest_int_wrapper) -FUNCTION_REFERENCE(wasm_f32x4_ceil, wasm::f32x4_ceil_wrapper) -FUNCTION_REFERENCE(wasm_f32x4_floor, wasm::f32x4_floor_wrapper) -FUNCTION_REFERENCE(wasm_f32x4_trunc, wasm::f32x4_trunc_wrapper) -FUNCTION_REFERENCE(wasm_f32x4_nearest_int, wasm::f32x4_nearest_int_wrapper) -FUNCTION_REFERENCE(wasm_memory_init, wasm::memory_init_wrapper) -FUNCTION_REFERENCE(wasm_memory_copy, wasm::memory_copy_wrapper) -FUNCTION_REFERENCE(wasm_memory_fill, wasm::memory_fill_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f32_trunc, wasm::f32_trunc_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f32_floor, wasm::f32_floor_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f32_ceil, wasm::f32_ceil_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f32_nearest_int, wasm::f32_nearest_int_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f64_trunc, wasm::f64_trunc_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f64_floor, wasm::f64_floor_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f64_ceil, wasm::f64_ceil_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f64_nearest_int, wasm::f64_nearest_int_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_int64_to_float32, + wasm::int64_to_float32_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_uint64_to_float32, + wasm::uint64_to_float32_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_int64_to_float64, + wasm::int64_to_float64_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_uint64_to_float64, + wasm::uint64_to_float64_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float32_to_int64, + wasm::float32_to_int64_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float32_to_uint64, + wasm::float32_to_uint64_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float64_to_int64, + wasm::float64_to_int64_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float64_to_uint64, + wasm::float64_to_uint64_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float32_to_int64_sat, + wasm::float32_to_int64_sat_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float32_to_uint64_sat, + wasm::float32_to_uint64_sat_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float64_to_int64_sat, + wasm::float64_to_int64_sat_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float64_to_uint64_sat, + wasm::float64_to_uint64_sat_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_int64_div, wasm::int64_div_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_int64_mod, wasm::int64_mod_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_uint64_div, wasm::uint64_div_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_uint64_mod, wasm::uint64_mod_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_word32_ctz, wasm::word32_ctz_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_word64_ctz, wasm::word64_ctz_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_word32_popcnt, wasm::word32_popcnt_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_word64_popcnt, wasm::word64_popcnt_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_word32_rol, wasm::word32_rol_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_word32_ror, wasm::word32_ror_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_word64_rol, wasm::word64_rol_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_word64_ror, wasm::word64_ror_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f64x2_ceil, wasm::f64x2_ceil_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f64x2_floor, wasm::f64x2_floor_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f64x2_trunc, wasm::f64x2_trunc_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f64x2_nearest_int, + wasm::f64x2_nearest_int_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f32x4_ceil, wasm::f32x4_ceil_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f32x4_floor, wasm::f32x4_floor_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f32x4_trunc, wasm::f32x4_trunc_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_f32x4_nearest_int, + wasm::f32x4_nearest_int_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_memory_init, wasm::memory_init_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_memory_copy, wasm::memory_copy_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_memory_fill, wasm::memory_fill_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_float64_pow, wasm::float64_pow_wrapper) +IF_WASM(FUNCTION_REFERENCE, wasm_call_trap_callback_for_testing, + wasm::call_trap_callback_for_testing) static void f64_acos_wrapper(Address data) { double input = ReadUnalignedValue<double>(data); @@ -397,7 +413,6 @@ static void f64_asin_wrapper(Address data) { FUNCTION_REFERENCE(f64_asin_wrapper_function, f64_asin_wrapper) -FUNCTION_REFERENCE(wasm_float64_pow, wasm::float64_pow_wrapper) static void f64_mod_wrapper(Address data) { double dividend = ReadUnalignedValue<double>(data); @@ -407,9 +422,6 @@ static void f64_mod_wrapper(Address data) { FUNCTION_REFERENCE(f64_mod_wrapper_function, f64_mod_wrapper) -FUNCTION_REFERENCE(wasm_call_trap_callback_for_testing, - wasm::call_trap_callback_for_testing) - ExternalReference ExternalReference::isolate_root(Isolate* isolate) { return ExternalReference(isolate->isolate_root()); } @@ -609,6 +621,34 @@ ExternalReference::address_of_enable_experimental_regexp_engine() { return ExternalReference(&FLAG_enable_experimental_regexp_engine); } +namespace { + +static uintptr_t BaselinePCForBytecodeOffset(Address raw_code_obj, + int bytecode_offset, + Address raw_bytecode_array) { + Code code_obj = Code::cast(Object(raw_code_obj)); + BytecodeArray bytecode_array = + BytecodeArray::cast(Object(raw_bytecode_array)); + return code_obj.GetBaselineStartPCForBytecodeOffset(bytecode_offset, + bytecode_array); +} + +static uintptr_t BaselinePCForNextExecutedBytecode(Address raw_code_obj, + int bytecode_offset, + Address raw_bytecode_array) { + Code code_obj = Code::cast(Object(raw_code_obj)); + BytecodeArray bytecode_array = + BytecodeArray::cast(Object(raw_bytecode_array)); + return code_obj.GetBaselinePCForNextExecutedBytecode(bytecode_offset, + bytecode_array); +} + +} // namespace + +FUNCTION_REFERENCE(baseline_pc_for_bytecode_offset, BaselinePCForBytecodeOffset) +FUNCTION_REFERENCE(baseline_pc_for_next_executed_bytecode, + BaselinePCForNextExecutedBytecode) + ExternalReference ExternalReference::thread_in_wasm_flag_address_address( Isolate* isolate) { return ExternalReference(isolate->thread_in_wasm_flag_address_address()); @@ -925,11 +965,6 @@ ExternalReference ExternalReference::cpu_features() { return ExternalReference(&CpuFeatures::supported_); } -ExternalReference ExternalReference::promise_hook_flags_address( - Isolate* isolate) { - return ExternalReference(isolate->promise_hook_flags_address()); -} - ExternalReference ExternalReference::promise_hook_address(Isolate* isolate) { return ExternalReference(isolate->promise_hook_address()); } @@ -939,6 +974,21 @@ ExternalReference ExternalReference::async_event_delegate_address( return ExternalReference(isolate->async_event_delegate_address()); } +ExternalReference +ExternalReference::promise_hook_or_async_event_delegate_address( + Isolate* isolate) { + return ExternalReference( + isolate->promise_hook_or_async_event_delegate_address()); +} + +ExternalReference ExternalReference:: + promise_hook_or_debug_is_active_or_async_event_delegate_address( + Isolate* isolate) { + return ExternalReference( + isolate + ->promise_hook_or_debug_is_active_or_async_event_delegate_address()); +} + ExternalReference ExternalReference::debug_execution_mode_address( Isolate* isolate) { return ExternalReference(isolate->debug_execution_mode_address()); diff --git a/deps/v8/src/codegen/external-reference.h b/deps/v8/src/codegen/external-reference.h index 0cd80ca6f1..f53db401c9 100644 --- a/deps/v8/src/codegen/external-reference.h +++ b/deps/v8/src/codegen/external-reference.h @@ -50,9 +50,13 @@ class StatsCounter; V(handle_scope_limit_address, "HandleScope::limit") \ V(scheduled_exception_address, "Isolate::scheduled_exception") \ V(address_of_pending_message_obj, "address_of_pending_message_obj") \ - V(promise_hook_flags_address, "Isolate::promise_hook_flags_address()") \ V(promise_hook_address, "Isolate::promise_hook_address()") \ V(async_event_delegate_address, "Isolate::async_event_delegate_address()") \ + V(promise_hook_or_async_event_delegate_address, \ + "Isolate::promise_hook_or_async_event_delegate_address()") \ + V(promise_hook_or_debug_is_active_or_async_event_delegate_address, \ + "Isolate::promise_hook_or_debug_is_active_or_async_event_delegate_" \ + "address()") \ V(debug_execution_mode_address, "Isolate::debug_execution_mode_address()") \ V(debug_is_active_address, "Debug::is_active_address()") \ V(debug_hook_on_function_call_address, \ @@ -120,6 +124,9 @@ class StatsCounter; V(address_of_wasm_i8x16_splat_0x33, "wasm_i8x16_splat_0x33") \ V(address_of_wasm_i8x16_splat_0x55, "wasm_i8x16_splat_0x55") \ V(address_of_wasm_i16x8_splat_0x0001, "wasm_16x8_splat_0x0001") \ + V(baseline_pc_for_bytecode_offset, "BaselinePCForBytecodeOffset") \ + V(baseline_pc_for_next_executed_bytecode, \ + "BaselinePCForNextExecutedBytecode") \ V(bytecode_size_table_address, "Bytecodes::bytecode_size_table_address") \ V(check_object_type, "check_object_type") \ V(compute_integer_hash, "ComputeSeededHash") \ @@ -194,52 +201,54 @@ class StatsCounter; V(string_to_array_index_function, "String::ToArrayIndex") \ V(try_string_to_index_or_lookup_existing, \ "try_string_to_index_or_lookup_existing") \ - V(wasm_call_trap_callback_for_testing, \ - "wasm::call_trap_callback_for_testing") \ - V(wasm_f32_ceil, "wasm::f32_ceil_wrapper") \ - V(wasm_f32_floor, "wasm::f32_floor_wrapper") \ - V(wasm_f32_nearest_int, "wasm::f32_nearest_int_wrapper") \ - V(wasm_f32_trunc, "wasm::f32_trunc_wrapper") \ - V(wasm_f64_ceil, "wasm::f64_ceil_wrapper") \ - V(wasm_f64_floor, "wasm::f64_floor_wrapper") \ - V(wasm_f64_nearest_int, "wasm::f64_nearest_int_wrapper") \ - V(wasm_f64_trunc, "wasm::f64_trunc_wrapper") \ - V(wasm_float32_to_int64, "wasm::float32_to_int64_wrapper") \ - V(wasm_float32_to_uint64, "wasm::float32_to_uint64_wrapper") \ - V(wasm_float32_to_int64_sat, "wasm::float32_to_int64_sat_wrapper") \ - V(wasm_float32_to_uint64_sat, "wasm::float32_to_uint64_sat_wrapper") \ - V(wasm_float64_pow, "wasm::float64_pow") \ - V(wasm_float64_to_int64, "wasm::float64_to_int64_wrapper") \ - V(wasm_float64_to_uint64, "wasm::float64_to_uint64_wrapper") \ - V(wasm_float64_to_int64_sat, "wasm::float64_to_int64_sat_wrapper") \ - V(wasm_float64_to_uint64_sat, "wasm::float64_to_uint64_sat_wrapper") \ - V(wasm_int64_div, "wasm::int64_div") \ - V(wasm_int64_mod, "wasm::int64_mod") \ - V(wasm_int64_to_float32, "wasm::int64_to_float32_wrapper") \ - V(wasm_int64_to_float64, "wasm::int64_to_float64_wrapper") \ - V(wasm_uint64_div, "wasm::uint64_div") \ - V(wasm_uint64_mod, "wasm::uint64_mod") \ - V(wasm_uint64_to_float32, "wasm::uint64_to_float32_wrapper") \ - V(wasm_uint64_to_float64, "wasm::uint64_to_float64_wrapper") \ - V(wasm_word32_ctz, "wasm::word32_ctz") \ - V(wasm_word32_popcnt, "wasm::word32_popcnt") \ - V(wasm_word32_rol, "wasm::word32_rol") \ - V(wasm_word32_ror, "wasm::word32_ror") \ - V(wasm_word64_rol, "wasm::word64_rol") \ - V(wasm_word64_ror, "wasm::word64_ror") \ - V(wasm_word64_ctz, "wasm::word64_ctz") \ - V(wasm_word64_popcnt, "wasm::word64_popcnt") \ - V(wasm_f64x2_ceil, "wasm::f64x2_ceil_wrapper") \ - V(wasm_f64x2_floor, "wasm::f64x2_floor_wrapper") \ - V(wasm_f64x2_trunc, "wasm::f64x2_trunc_wrapper") \ - V(wasm_f64x2_nearest_int, "wasm::f64x2_nearest_int_wrapper") \ - V(wasm_f32x4_ceil, "wasm::f32x4_ceil_wrapper") \ - V(wasm_f32x4_floor, "wasm::f32x4_floor_wrapper") \ - V(wasm_f32x4_trunc, "wasm::f32x4_trunc_wrapper") \ - V(wasm_f32x4_nearest_int, "wasm::f32x4_nearest_int_wrapper") \ - V(wasm_memory_init, "wasm::memory_init") \ - V(wasm_memory_copy, "wasm::memory_copy") \ - V(wasm_memory_fill, "wasm::memory_fill") \ + IF_WASM(V, wasm_call_trap_callback_for_testing, \ + "wasm::call_trap_callback_for_testing") \ + IF_WASM(V, wasm_f32_ceil, "wasm::f32_ceil_wrapper") \ + IF_WASM(V, wasm_f32_floor, "wasm::f32_floor_wrapper") \ + IF_WASM(V, wasm_f32_nearest_int, "wasm::f32_nearest_int_wrapper") \ + IF_WASM(V, wasm_f32_trunc, "wasm::f32_trunc_wrapper") \ + IF_WASM(V, wasm_f64_ceil, "wasm::f64_ceil_wrapper") \ + IF_WASM(V, wasm_f64_floor, "wasm::f64_floor_wrapper") \ + IF_WASM(V, wasm_f64_nearest_int, "wasm::f64_nearest_int_wrapper") \ + IF_WASM(V, wasm_f64_trunc, "wasm::f64_trunc_wrapper") \ + IF_WASM(V, wasm_float32_to_int64, "wasm::float32_to_int64_wrapper") \ + IF_WASM(V, wasm_float32_to_uint64, "wasm::float32_to_uint64_wrapper") \ + IF_WASM(V, wasm_float32_to_int64_sat, "wasm::float32_to_int64_sat_wrapper") \ + IF_WASM(V, wasm_float32_to_uint64_sat, \ + "wasm::float32_to_uint64_sat_wrapper") \ + IF_WASM(V, wasm_float64_pow, "wasm::float64_pow") \ + IF_WASM(V, wasm_float64_to_int64, "wasm::float64_to_int64_wrapper") \ + IF_WASM(V, wasm_float64_to_uint64, "wasm::float64_to_uint64_wrapper") \ + IF_WASM(V, wasm_float64_to_int64_sat, "wasm::float64_to_int64_sat_wrapper") \ + IF_WASM(V, wasm_float64_to_uint64_sat, \ + "wasm::float64_to_uint64_sat_wrapper") \ + IF_WASM(V, wasm_int64_div, "wasm::int64_div") \ + IF_WASM(V, wasm_int64_mod, "wasm::int64_mod") \ + IF_WASM(V, wasm_int64_to_float32, "wasm::int64_to_float32_wrapper") \ + IF_WASM(V, wasm_int64_to_float64, "wasm::int64_to_float64_wrapper") \ + IF_WASM(V, wasm_uint64_div, "wasm::uint64_div") \ + IF_WASM(V, wasm_uint64_mod, "wasm::uint64_mod") \ + IF_WASM(V, wasm_uint64_to_float32, "wasm::uint64_to_float32_wrapper") \ + IF_WASM(V, wasm_uint64_to_float64, "wasm::uint64_to_float64_wrapper") \ + IF_WASM(V, wasm_word32_ctz, "wasm::word32_ctz") \ + IF_WASM(V, wasm_word32_popcnt, "wasm::word32_popcnt") \ + IF_WASM(V, wasm_word32_rol, "wasm::word32_rol") \ + IF_WASM(V, wasm_word32_ror, "wasm::word32_ror") \ + IF_WASM(V, wasm_word64_rol, "wasm::word64_rol") \ + IF_WASM(V, wasm_word64_ror, "wasm::word64_ror") \ + IF_WASM(V, wasm_word64_ctz, "wasm::word64_ctz") \ + IF_WASM(V, wasm_word64_popcnt, "wasm::word64_popcnt") \ + IF_WASM(V, wasm_f64x2_ceil, "wasm::f64x2_ceil_wrapper") \ + IF_WASM(V, wasm_f64x2_floor, "wasm::f64x2_floor_wrapper") \ + IF_WASM(V, wasm_f64x2_trunc, "wasm::f64x2_trunc_wrapper") \ + IF_WASM(V, wasm_f64x2_nearest_int, "wasm::f64x2_nearest_int_wrapper") \ + IF_WASM(V, wasm_f32x4_ceil, "wasm::f32x4_ceil_wrapper") \ + IF_WASM(V, wasm_f32x4_floor, "wasm::f32x4_floor_wrapper") \ + IF_WASM(V, wasm_f32x4_trunc, "wasm::f32x4_trunc_wrapper") \ + IF_WASM(V, wasm_f32x4_nearest_int, "wasm::f32x4_nearest_int_wrapper") \ + IF_WASM(V, wasm_memory_init, "wasm::memory_init") \ + IF_WASM(V, wasm_memory_copy, "wasm::memory_copy") \ + IF_WASM(V, wasm_memory_fill, "wasm::memory_fill") \ V(address_of_wasm_f64x2_convert_low_i32x4_u_int_mask, \ "wasm_f64x2_convert_low_i32x4_u_int_mask") \ V(supports_wasm_simd_128_address, "wasm::supports_wasm_simd_128_address") \ diff --git a/deps/v8/src/codegen/handler-table.cc b/deps/v8/src/codegen/handler-table.cc index 8aec047d13..7bede6aa9b 100644 --- a/deps/v8/src/codegen/handler-table.cc +++ b/deps/v8/src/codegen/handler-table.cc @@ -11,7 +11,10 @@ #include "src/codegen/assembler-inl.h" #include "src/objects/code-inl.h" #include "src/objects/objects-inl.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY namespace v8 { namespace internal { @@ -20,9 +23,11 @@ HandlerTable::HandlerTable(Code code) : HandlerTable(code.HandlerTableAddress(), code.handler_table_size(), kReturnAddressBasedEncoding) {} +#if V8_ENABLE_WEBASSEMBLY HandlerTable::HandlerTable(const wasm::WasmCode* code) : HandlerTable(code->handler_table(), code->handler_table_size(), kReturnAddressBasedEncoding) {} +#endif // V8_ENABLE_WEBASSEMBLY HandlerTable::HandlerTable(BytecodeArray bytecode_array) : HandlerTable(bytecode_array.handler_table()) {} diff --git a/deps/v8/src/codegen/handler-table.h b/deps/v8/src/codegen/handler-table.h index d2ed80a0e0..e1626e2be5 100644 --- a/deps/v8/src/codegen/handler-table.h +++ b/deps/v8/src/codegen/handler-table.h @@ -58,7 +58,9 @@ class V8_EXPORT_PRIVATE HandlerTable { // Constructors for the various encodings. explicit HandlerTable(Code code); explicit HandlerTable(ByteArray byte_array); +#if V8_ENABLE_WEBASSEMBLY explicit HandlerTable(const wasm::WasmCode* code); +#endif // V8_ENABLE_WEBASSEMBLY explicit HandlerTable(BytecodeArray bytecode_array); HandlerTable(Address handler_table, int handler_table_size, EncodingMode encoding_mode); diff --git a/deps/v8/src/codegen/ia32/assembler-ia32-inl.h b/deps/v8/src/codegen/ia32/assembler-ia32-inl.h index 25d2d486ce..1585f970e8 100644 --- a/deps/v8/src/codegen/ia32/assembler-ia32-inl.h +++ b/deps/v8/src/codegen/ia32/assembler-ia32-inl.h @@ -49,12 +49,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return true; } -bool CpuFeatures::SupportsWasmSimd128() { - if (IsSupported(SSE4_1)) return true; - if (FLAG_wasm_simd_ssse3_codegen && IsSupported(SSSE3)) return true; - return false; -} - // The modes possibly affected by apply must be in kApplyMask. void RelocInfo::apply(intptr_t delta) { DCHECK_EQ(kApplyMask, (RelocInfo::ModeMask(RelocInfo::CODE_TARGET) | diff --git a/deps/v8/src/codegen/ia32/assembler-ia32.cc b/deps/v8/src/codegen/ia32/assembler-ia32.cc index 3f9d7ddfa2..809df1daef 100644 --- a/deps/v8/src/codegen/ia32/assembler-ia32.cc +++ b/deps/v8/src/codegen/ia32/assembler-ia32.cc @@ -122,6 +122,14 @@ bool OSHasAVXSupport() { } // namespace +bool CpuFeatures::SupportsWasmSimd128() { +#if V8_ENABLE_WEBASSEMBLY + if (IsSupported(SSE4_1)) return true; + if (FLAG_wasm_simd_ssse3_codegen && IsSupported(SSSE3)) return true; +#endif // V8_ENABLE_WEBASSEMBLY + return false; +} + void CpuFeatures::ProbeImpl(bool cross_compile) { base::CPU cpu; CHECK(cpu.has_sse2()); // SSE2 support is mandatory. @@ -130,14 +138,9 @@ void CpuFeatures::ProbeImpl(bool cross_compile) { // Only use statically determined features for cross compile (snapshot). if (cross_compile) return; - if (cpu.has_sse42() && FLAG_enable_sse4_2) supported_ |= 1u << SSE4_2; - if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1; - if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3; - if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3; - if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() && - OSHasAVXSupport()) { - supported_ |= 1u << AVX; - } + // To deal with any combination of flags (e.g. --no-enable-sse4-1 + // --enable-sse-4-2), we start checking from the "highest" supported + // extension, for each extension, enable if newer extension is supported. if (cpu.has_avx2() && FLAG_enable_avx2 && IsSupported(AVX)) { supported_ |= 1u << AVX2; } @@ -145,6 +148,19 @@ void CpuFeatures::ProbeImpl(bool cross_compile) { OSHasAVXSupport()) { supported_ |= 1u << FMA3; } + if ((cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() && + OSHasAVXSupport()) || + IsSupported(AVX2) || IsSupported(FMA3)) { + supported_ |= 1u << AVX; + } + if ((cpu.has_sse42() && FLAG_enable_sse4_2) || IsSupported(AVX)) + supported_ |= 1u << SSE4_2; + if ((cpu.has_sse41() && FLAG_enable_sse4_1) || IsSupported(SSE4_2)) + supported_ |= 1u << SSE4_1; + if ((cpu.has_ssse3() && FLAG_enable_ssse3) || IsSupported(SSE4_1)) + supported_ |= 1u << SSSE3; + if ((cpu.has_sse3() && FLAG_enable_sse3) || IsSupported(SSSE3)) + supported_ |= 1u << SSE3; if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1; if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2; if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT; diff --git a/deps/v8/src/codegen/ia32/assembler-ia32.h b/deps/v8/src/codegen/ia32/assembler-ia32.h index 3914c35544..2a8fd3ee28 100644 --- a/deps/v8/src/codegen/ia32/assembler-ia32.h +++ b/deps/v8/src/codegen/ia32/assembler-ia32.h @@ -235,6 +235,12 @@ class V8_EXPORT_PRIVATE Operand { explicit Operand(Register base, int32_t disp, RelocInfo::Mode rmode = RelocInfo::NONE); + // [rip + disp/r] + explicit Operand(Label* label) { + set_modrm(0, ebp); + set_dispr(reinterpret_cast<intptr_t>(label), RelocInfo::INTERNAL_REFERENCE); + } + // [base + index*scale + disp/r] explicit Operand(Register base, Register index, ScaleFactor scale, int32_t disp, RelocInfo::Mode rmode = RelocInfo::NONE); diff --git a/deps/v8/src/codegen/ia32/interface-descriptors-ia32.cc b/deps/v8/src/codegen/ia32/interface-descriptors-ia32.cc index d732fa2fbb..fd76e01590 100644 --- a/deps/v8/src/codegen/ia32/interface-descriptors-ia32.cc +++ b/deps/v8/src/codegen/ia32/interface-descriptors-ia32.cc @@ -90,13 +90,9 @@ const Register GrowArrayElementsDescriptor::ObjectRegister() { return eax; } const Register GrowArrayElementsDescriptor::KeyRegister() { return ecx; } const Register BaselineLeaveFrameDescriptor::ParamsSizeRegister() { - // TODO(v8:11421): Implement on this platform. - UNREACHABLE(); -} -const Register BaselineLeaveFrameDescriptor::WeightRegister() { - // TODO(v8:11421): Implement on this platform. - UNREACHABLE(); + return esi; } +const Register BaselineLeaveFrameDescriptor::WeightRegister() { return edi; } // static const Register TypeConversionDescriptor::ArgumentRegister() { return eax; } @@ -224,8 +220,8 @@ void CompareDescriptor::InitializePlatformSpecific( void Compare_BaselineDescriptor::InitializePlatformSpecific( CallInterfaceDescriptorData* data) { - // TODO(v8:11421): Implement on this platform. - InitializePlatformUnimplemented(data, kParameterCount); + Register registers[] = {edx, eax, ecx}; + data->InitializePlatformSpecific(arraysize(registers), registers); } void BinaryOpDescriptor::InitializePlatformSpecific( @@ -236,8 +232,8 @@ void BinaryOpDescriptor::InitializePlatformSpecific( void BinaryOp_BaselineDescriptor::InitializePlatformSpecific( CallInterfaceDescriptorData* data) { - // TODO(v8:11421): Implement on this platform. - InitializePlatformUnimplemented(data, kParameterCount); + Register registers[] = {edx, eax, ecx}; + data->InitializePlatformSpecific(arraysize(registers), registers); } void ApiCallbackDescriptor::InitializePlatformSpecific( diff --git a/deps/v8/src/codegen/ia32/macro-assembler-ia32.cc b/deps/v8/src/codegen/ia32/macro-assembler-ia32.cc index 7a99d6c701..9892eb9470 100644 --- a/deps/v8/src/codegen/ia32/macro-assembler-ia32.cc +++ b/deps/v8/src/codegen/ia32/macro-assembler-ia32.cc @@ -628,28 +628,6 @@ void TurboAssembler::Cvttsd2ui(Register dst, Operand src, XMMRegister tmp) { add(dst, Immediate(0x80000000)); } -void TurboAssembler::Roundps(XMMRegister dst, XMMRegister src, - RoundingMode mode) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vroundps(dst, src, mode); - } else { - CpuFeatureScope scope(this, SSE4_1); - roundps(dst, src, mode); - } -} - -void TurboAssembler::Roundpd(XMMRegister dst, XMMRegister src, - RoundingMode mode) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vroundpd(dst, src, mode); - } else { - CpuFeatureScope scope(this, SSE4_1); - roundpd(dst, src, mode); - } -} - void TurboAssembler::Pmulhrsw(XMMRegister dst, XMMRegister src1, XMMRegister src2) { if (CpuFeatures::IsSupported(AVX)) { @@ -657,217 +635,13 @@ void TurboAssembler::Pmulhrsw(XMMRegister dst, XMMRegister src1, vpmulhrsw(dst, src1, src2); } else { if (dst != src1) { - movdqu(dst, src1); + movaps(dst, src1); } CpuFeatureScope sse_scope(this, SSSE3); pmulhrsw(dst, src2); } } -// 1. Unpack src0, src1 into even-number elements of scratch. -// 2. Unpack src1, src0 into even-number elements of dst. -// 3. Multiply 1. with 2. -// For non-AVX, use non-destructive pshufd instead of punpckldq/punpckhdq. -void TurboAssembler::I64x2ExtMul(XMMRegister dst, XMMRegister src1, - XMMRegister src2, XMMRegister scratch, - bool low, bool is_signed) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - if (low) { - vpunpckldq(scratch, src1, src1); - vpunpckldq(dst, src2, src2); - } else { - vpunpckhdq(scratch, src1, src1); - vpunpckhdq(dst, src2, src2); - } - if (is_signed) { - vpmuldq(dst, scratch, dst); - } else { - vpmuludq(dst, scratch, dst); - } - } else { - uint8_t mask = low ? 0x50 : 0xFA; - pshufd(scratch, src1, mask); - pshufd(dst, src2, mask); - if (is_signed) { - CpuFeatureScope sse4_scope(this, SSE4_1); - pmuldq(dst, scratch); - } else { - pmuludq(dst, scratch); - } - } -} - -// 1. Multiply low word into scratch. -// 2. Multiply high word (can be signed or unsigned) into dst. -// 3. Unpack and interleave scratch and dst into dst. -void TurboAssembler::I32x4ExtMul(XMMRegister dst, XMMRegister src1, - XMMRegister src2, XMMRegister scratch, - bool low, bool is_signed) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpmullw(scratch, src1, src2); - is_signed ? vpmulhw(dst, src1, src2) : vpmulhuw(dst, src1, src2); - low ? vpunpcklwd(dst, scratch, dst) : vpunpckhwd(dst, scratch, dst); - } else { - DCHECK_EQ(dst, src1); - movdqu(scratch, src1); - pmullw(dst, src2); - is_signed ? pmulhw(scratch, src2) : pmulhuw(scratch, src2); - low ? punpcklwd(dst, scratch) : punpckhwd(dst, scratch); - } -} - -void TurboAssembler::I16x8ExtMul(XMMRegister dst, XMMRegister src1, - XMMRegister src2, XMMRegister scratch, - bool low, bool is_signed) { - if (low) { - is_signed ? Pmovsxbw(scratch, src1) : Pmovzxbw(scratch, src1); - is_signed ? Pmovsxbw(dst, src2) : Pmovzxbw(dst, src2); - Pmullw(dst, scratch); - } else { - Palignr(scratch, src1, uint8_t{8}); - is_signed ? Pmovsxbw(scratch, scratch) : Pmovzxbw(scratch, scratch); - Palignr(dst, src2, uint8_t{8}); - is_signed ? Pmovsxbw(dst, dst) : Pmovzxbw(dst, dst); - Pmullw(dst, scratch); - } -} - -void TurboAssembler::S128Select(XMMRegister dst, XMMRegister mask, - XMMRegister src1, XMMRegister src2, - XMMRegister scratch) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpandn(scratch, mask, src2); - vpand(dst, src1, mask); - vpor(dst, dst, scratch); - } else { - DCHECK_EQ(dst, mask); - // Use float ops as they are 1 byte shorter than int ops. - movaps(scratch, dst); - andnps(scratch, src2); - andps(dst, src1); - orps(dst, scratch); - } -} - -void TurboAssembler::I64x2SConvertI32x4High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpunpckhqdq(dst, src, src); - vpmovsxdq(dst, dst); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - pshufd(dst, src, 0xEE); - pmovsxdq(dst, dst); - } -} - -void TurboAssembler::I64x2UConvertI32x4High(XMMRegister dst, XMMRegister src, - XMMRegister scratch) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpxor(scratch, scratch, scratch); - vpunpckhdq(dst, src, scratch); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - pshufd(dst, src, 0xEE); - pmovzxdq(dst, dst); - } -} - -void TurboAssembler::I32x4SConvertI16x8High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - // src = |a|b|c|d|e|f|g|h| (high) - // dst = |e|e|f|f|g|g|h|h| - vpunpckhwd(dst, src, src); - vpsrad(dst, dst, 16); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - if (dst == src) { - // 2 bytes shorter than pshufd, but has depdency on dst. - movhlps(dst, src); - pmovsxwd(dst, dst); - } else { - // No dependency on dst. - pshufd(dst, src, 0xEE); - pmovsxwd(dst, dst); - } - } -} - -void TurboAssembler::I32x4UConvertI16x8High(XMMRegister dst, XMMRegister src, - XMMRegister scratch) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - // scratch = |0|0|0|0|0|0|0|0| - // src = |a|b|c|d|e|f|g|h| - // dst = |0|a|0|b|0|c|0|d| - XMMRegister tmp = dst == src ? scratch : dst; - vpxor(tmp, tmp, tmp); - vpunpckhwd(dst, src, tmp); - } else { - if (dst == src) { - // xorps can be executed on more ports than pshufd. - xorps(scratch, scratch); - punpckhwd(dst, scratch); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - // No dependency on dst. - pshufd(dst, src, 0xEE); - pmovzxwd(dst, dst); - } - } -} - -void TurboAssembler::I16x8SConvertI8x16High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - // src = |a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p| (high) - // dst = |i|i|j|j|k|k|l|l|m|m|n|n|o|o|p|p| - vpunpckhbw(dst, src, src); - vpsraw(dst, dst, 8); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - if (dst == src) { - // 2 bytes shorter than pshufd, but has depdency on dst. - movhlps(dst, src); - pmovsxbw(dst, dst); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - // No dependency on dst. - pshufd(dst, src, 0xEE); - pmovsxbw(dst, dst); - } - } -} - -void TurboAssembler::I16x8UConvertI8x16High(XMMRegister dst, XMMRegister src, - XMMRegister scratch) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - // tmp = |0|0|0|0|0|0|0|0 | 0|0|0|0|0|0|0|0| - // src = |a|b|c|d|e|f|g|h | i|j|k|l|m|n|o|p| - // dst = |0|a|0|b|0|c|0|d | 0|e|0|f|0|g|0|h| - XMMRegister tmp = dst == src ? scratch : dst; - vpxor(tmp, tmp, tmp); - vpunpckhbw(dst, src, tmp); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - if (dst == src) { - // xorps can be executed on more ports than pshufd. - xorps(scratch, scratch); - punpckhbw(dst, scratch); - } else { - // No dependency on dst. - pshufd(dst, src, 0xEE); - pmovzxbw(dst, dst); - } - } -} - void TurboAssembler::I16x8Q15MulRSatS(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister scratch) { // k = i16x8.splat(0x8000) @@ -879,16 +653,6 @@ void TurboAssembler::I16x8Q15MulRSatS(XMMRegister dst, XMMRegister src1, Pxor(dst, scratch); } -void TurboAssembler::S128Store32Lane(Operand dst, XMMRegister src, - uint8_t laneidx) { - if (laneidx == 0) { - Movss(dst, src); - } else { - DCHECK_GE(3, laneidx); - Extractps(dst, src, laneidx); - } -} - void TurboAssembler::I8x16Popcnt(XMMRegister dst, XMMRegister src, XMMRegister tmp1, XMMRegister tmp2, Register scratch) { @@ -915,8 +679,8 @@ void TurboAssembler::I8x16Popcnt(XMMRegister dst, XMMRegister src, // PSHUFB instruction, thus use PSHUFB-free divide-and-conquer // algorithm on these processors. ATOM CPU feature captures exactly // the right set of processors. - xorps(tmp1, tmp1); - pavgb(tmp1, src); + movaps(tmp1, src); + psrlw(tmp1, 1); if (dst != src) { movaps(dst, src); } @@ -963,6 +727,10 @@ void TurboAssembler::F64x2ConvertLowI32x4U(XMMRegister dst, XMMRegister src, // dst = [ src_low, 0x43300000, src_high, 0x4330000 ]; // 0x43300000'00000000 is a special double where the significand bits // precisely represents all uint32 numbers. + if (!CpuFeatures::IsSupported(AVX) && dst != src) { + movaps(dst, src); + src = dst; + } Unpcklps(dst, src, ExternalReferenceAsOperand( ExternalReference:: @@ -1053,85 +821,6 @@ void TurboAssembler::I32x4TruncSatF64x2UZero(XMMRegister dst, XMMRegister src, } } -void TurboAssembler::I64x2Abs(XMMRegister dst, XMMRegister src, - XMMRegister scratch) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - XMMRegister tmp = dst == src ? scratch : dst; - vpxor(tmp, tmp, tmp); - vpsubq(tmp, tmp, src); - vblendvpd(dst, src, tmp, src); - } else { - CpuFeatureScope sse_scope(this, SSE3); - movshdup(scratch, src); - if (dst != src) { - movaps(dst, src); - } - psrad(scratch, 31); - xorps(dst, scratch); - psubq(dst, scratch); - } -} - -void TurboAssembler::I64x2GtS(XMMRegister dst, XMMRegister src0, - XMMRegister src1, XMMRegister scratch) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpcmpgtq(dst, src0, src1); - } else if (CpuFeatures::IsSupported(SSE4_2)) { - CpuFeatureScope sse_scope(this, SSE4_2); - DCHECK_EQ(dst, src0); - pcmpgtq(dst, src1); - } else { - CpuFeatureScope sse_scope(this, SSSE3); - DCHECK_NE(dst, src0); - DCHECK_NE(dst, src1); - movaps(dst, src1); - movaps(scratch, src0); - psubq(dst, src0); - pcmpeqd(scratch, src1); - andps(dst, scratch); - movaps(scratch, src0); - pcmpgtd(scratch, src1); - orps(dst, scratch); - movshdup(dst, dst); - } -} - -void TurboAssembler::I64x2GeS(XMMRegister dst, XMMRegister src0, - XMMRegister src1, XMMRegister scratch) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpcmpgtq(dst, src1, src0); - vpcmpeqd(scratch, scratch, scratch); - vpxor(dst, dst, scratch); - } else if (CpuFeatures::IsSupported(SSE4_2)) { - CpuFeatureScope sse_scope(this, SSE4_2); - DCHECK_NE(dst, src0); - if (dst != src1) { - movaps(dst, src1); - } - pcmpgtq(dst, src0); - pcmpeqd(scratch, scratch); - xorps(dst, scratch); - } else { - CpuFeatureScope sse_scope(this, SSSE3); - DCHECK_NE(dst, src0); - DCHECK_NE(dst, src1); - movaps(dst, src0); - movaps(scratch, src1); - psubq(dst, src1); - pcmpeqd(scratch, src0); - andps(dst, scratch); - movaps(scratch, src1); - pcmpgtd(scratch, src0); - orps(dst, scratch); - movshdup(dst, dst); - pcmpeqd(scratch, scratch); - xorps(dst, scratch); - } -} - void TurboAssembler::I16x8ExtAddPairwiseI8x16S(XMMRegister dst, XMMRegister src, XMMRegister tmp, Register scratch) { @@ -1160,19 +849,20 @@ void TurboAssembler::I16x8ExtAddPairwiseI8x16U(XMMRegister dst, XMMRegister src, Register scratch) { Operand op = ExternalReferenceAsOperand( ExternalReference::address_of_wasm_i8x16_splat_0x01(), scratch); - if (!CpuFeatures::IsSupported(AVX) && dst != src) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpmaddubsw(dst, src, op); + } else { + CpuFeatureScope sse_scope(this, SSSE3); movaps(dst, src); + pmaddubsw(dst, op); } - Pmaddubsw(dst, src, op); } void TurboAssembler::I32x4ExtAddPairwiseI16x8S(XMMRegister dst, XMMRegister src, Register scratch) { Operand op = ExternalReferenceAsOperand( ExternalReference::address_of_wasm_i16x8_splat_0x0001(), scratch); - if (!CpuFeatures::IsSupported(AVX) && dst != src) { - movaps(dst, src); - } // pmaddwd multiplies signed words in src and op, producing // signed doublewords, then adds pairwise. // src = |a|b|c|d|e|f|g|h| @@ -1182,16 +872,68 @@ void TurboAssembler::I32x4ExtAddPairwiseI16x8S(XMMRegister dst, XMMRegister src, void TurboAssembler::I32x4ExtAddPairwiseI16x8U(XMMRegister dst, XMMRegister src, XMMRegister tmp) { - // src = |a|b|c|d|e|f|g|h| - // tmp = i32x4.splat(0x0000FFFF) - Pcmpeqd(tmp, tmp); - Psrld(tmp, tmp, byte{16}); - // tmp =|0|b|0|d|0|f|0|h| - Pand(tmp, src); - // dst = |0|a|0|c|0|e|0|g| - Psrld(dst, src, byte{16}); - // dst = |a+b|c+d|e+f|g+h| - Paddd(dst, dst, tmp); + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + // src = |a|b|c|d|e|f|g|h| (low) + // scratch = |0|a|0|c|0|e|0|g| + vpsrld(tmp, src, 16); + // dst = |0|b|0|d|0|f|0|h| + vpblendw(dst, src, tmp, 0xAA); + // dst = |a+b|c+d|e+f|g+h| + vpaddd(dst, tmp, dst); + } else if (CpuFeatures::IsSupported(SSE4_1)) { + CpuFeatureScope sse_scope(this, SSE4_1); + // There is a potentially better lowering if we get rip-relative constants, + // see https://github.com/WebAssembly/simd/pull/380. + movaps(tmp, src); + psrld(tmp, 16); + if (dst != src) { + movaps(dst, src); + } + pblendw(dst, tmp, 0xAA); + paddd(dst, tmp); + } else { + // src = |a|b|c|d|e|f|g|h| + // tmp = i32x4.splat(0x0000FFFF) + pcmpeqd(tmp, tmp); + psrld(tmp, byte{16}); + // tmp =|0|b|0|d|0|f|0|h| + andps(tmp, src); + // dst = |0|a|0|c|0|e|0|g| + if (dst != src) { + movaps(dst, src); + } + psrld(dst, byte{16}); + // dst = |a+b|c+d|e+f|g+h| + paddd(dst, tmp); + } +} + +void TurboAssembler::I8x16Swizzle(XMMRegister dst, XMMRegister src, + XMMRegister mask, XMMRegister scratch, + Register tmp, bool omit_add) { + if (omit_add) { + Pshufb(dst, src, mask); + return; + } + + // Out-of-range indices should return 0, add 112 so that any value > 15 + // saturates to 128 (top bit set), so pshufb will zero that lane. + Operand op = ExternalReferenceAsOperand( + ExternalReference::address_of_wasm_i8x16_swizzle_mask(), tmp); + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpaddusb(scratch, mask, op); + vpshufb(dst, src, scratch); + } else { + CpuFeatureScope sse_scope(this, SSSE3); + movaps(scratch, op); + if (dst != src) { + movaps(dst, src); + } + paddusb(scratch, mask); + pshufb(dst, scratch); + } } void TurboAssembler::ShlPair(Register high, Register low, uint8_t shift) { @@ -1399,11 +1141,13 @@ void TurboAssembler::Prologue() { void TurboAssembler::EnterFrame(StackFrame::Type type) { push(ebp); mov(ebp, esp); - push(Immediate(StackFrame::TypeToMarker(type))); + if (!StackFrame::IsJavaScript(type)) { + Push(Immediate(StackFrame::TypeToMarker(type))); + } } void TurboAssembler::LeaveFrame(StackFrame::Type type) { - if (emit_debug_code()) { + if (emit_debug_code() && !StackFrame::IsJavaScript(type)) { cmp(Operand(ebp, CommonFrameConstants::kContextOrFrameTypeOffset), Immediate(StackFrame::TypeToMarker(type))); Check(equal, AbortReason::kStackFrameTypesMustMatch); @@ -2019,6 +1763,8 @@ void TurboAssembler::Move(Operand dst, const Immediate& src) { } } +void TurboAssembler::Move(Register dst, Operand src) { mov(dst, src); } + void TurboAssembler::Move(Register dst, Handle<HeapObject> src) { if (root_array_available() && options().isolate_independent_code) { IndirectLoadConstant(dst, src); @@ -2090,152 +1836,6 @@ void TurboAssembler::Move(XMMRegister dst, uint64_t src) { } } -void TurboAssembler::Cmpeqps(XMMRegister dst, XMMRegister src1, - XMMRegister src2) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vcmpeqps(dst, src1, src2); - } else { - movaps(dst, src1); - cmpeqps(dst, src2); - } -} - -void TurboAssembler::Pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpshufhw(dst, src, shuffle); - } else { - pshufhw(dst, src, shuffle); - } -} - -void TurboAssembler::Pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpshuflw(dst, src, shuffle); - } else { - pshuflw(dst, src, shuffle); - } -} - -void TurboAssembler::Pshufd(XMMRegister dst, Operand src, uint8_t shuffle) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpshufd(dst, src, shuffle); - } else { - pshufd(dst, src, shuffle); - } -} - -void TurboAssembler::Psraw(XMMRegister dst, uint8_t shift) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpsraw(dst, dst, shift); - } else { - psraw(dst, shift); - } -} - -void TurboAssembler::Psrlw(XMMRegister dst, uint8_t shift) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpsrlw(dst, dst, shift); - } else { - psrlw(dst, shift); - } -} - -void TurboAssembler::Psrlq(XMMRegister dst, uint8_t shift) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpsrlq(dst, dst, shift); - } else { - psrlq(dst, shift); - } -} - -void TurboAssembler::Psignb(XMMRegister dst, Operand src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpsignb(dst, dst, src); - return; - } - if (CpuFeatures::IsSupported(SSSE3)) { - CpuFeatureScope sse_scope(this, SSSE3); - psignb(dst, src); - return; - } - FATAL("no AVX or SSE3 support"); -} - -void TurboAssembler::Psignw(XMMRegister dst, Operand src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpsignw(dst, dst, src); - return; - } - if (CpuFeatures::IsSupported(SSSE3)) { - CpuFeatureScope sse_scope(this, SSSE3); - psignw(dst, src); - return; - } - FATAL("no AVX or SSE3 support"); -} - -void TurboAssembler::Psignd(XMMRegister dst, Operand src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpsignd(dst, dst, src); - return; - } - if (CpuFeatures::IsSupported(SSSE3)) { - CpuFeatureScope sse_scope(this, SSSE3); - psignd(dst, src); - return; - } - FATAL("no AVX or SSE3 support"); -} - -void TurboAssembler::Haddps(XMMRegister dst, XMMRegister src1, Operand src2) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vhaddps(dst, src1, src2); - } else { - CpuFeatureScope scope(this, SSE3); - DCHECK_EQ(dst, src1); - haddps(dst, src2); - } -} - -void TurboAssembler::Pcmpeqq(XMMRegister dst, Operand src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpcmpeqq(dst, dst, src); - } else { - CpuFeatureScope scope(this, SSE4_1); - pcmpeqq(dst, src); - } -} - -void TurboAssembler::Pcmpeqq(XMMRegister dst, XMMRegister src1, - XMMRegister src2) { - Pcmpeqq(dst, src1, Operand(src2)); -} - -void TurboAssembler::Pcmpeqq(XMMRegister dst, XMMRegister src1, Operand src2) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpcmpeqq(dst, src1, src2); - } else { - // pcmpeqq is only used by Wasm SIMD, which requires SSE4_1. - DCHECK(CpuFeatures::IsSupported(SSE4_1)); - CpuFeatureScope scope(this, SSE4_1); - DCHECK_EQ(dst, src1); - pcmpeqq(dst, src2); - } -} - void TurboAssembler::Pshufb(XMMRegister dst, XMMRegister src, Operand mask) { if (CpuFeatures::IsSupported(AVX)) { CpuFeatureScope scope(this, AVX); @@ -2280,58 +1880,6 @@ void TurboAssembler::Palignr(XMMRegister dst, Operand src, uint8_t imm8) { FATAL("no AVX or SSE3 support"); } -void TurboAssembler::Pextrb(Operand dst, XMMRegister src, uint8_t imm8) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpextrb(dst, src, imm8); - return; - } - DCHECK(CpuFeatures::IsSupported(SSE4_1)); - CpuFeatureScope sse_scope(this, SSE4_1); - pextrb(dst, src, imm8); - return; -} - -void TurboAssembler::Pextrb(Register dst, XMMRegister src, uint8_t imm8) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpextrb(dst, src, imm8); - return; - } - if (CpuFeatures::IsSupported(SSE4_1)) { - CpuFeatureScope sse_scope(this, SSE4_1); - pextrb(dst, src, imm8); - return; - } - FATAL("no AVX or SSE4.1 support"); -} - -void TurboAssembler::Pextrw(Operand dst, XMMRegister src, uint8_t imm8) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpextrw(dst, src, imm8); - return; - } - DCHECK(CpuFeatures::IsSupported(SSE4_1)); - CpuFeatureScope sse_scope(this, SSE4_1); - pextrw(dst, src, imm8); - return; -} - -void TurboAssembler::Pextrw(Register dst, XMMRegister src, uint8_t imm8) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpextrw(dst, src, imm8); - return; - } - if (CpuFeatures::IsSupported(SSE4_1)) { - CpuFeatureScope sse_scope(this, SSE4_1); - pextrw(dst, src, imm8); - return; - } - FATAL("no AVX or SSE4.1 support"); -} - void TurboAssembler::Pextrd(Register dst, XMMRegister src, uint8_t imm8) { if (imm8 == 0) { Movd(dst, src); @@ -2371,7 +1919,7 @@ void TurboAssembler::Pinsrb(XMMRegister dst, XMMRegister src1, Operand src2, if (CpuFeatures::IsSupported(SSE4_1)) { CpuFeatureScope sse_scope(this, SSE4_1); if (dst != src1) { - movdqu(dst, src1); + movaps(dst, src1); } pinsrb(dst, src2, imm8); return; @@ -2387,7 +1935,7 @@ void TurboAssembler::Pinsrd(XMMRegister dst, XMMRegister src1, Operand src2, return; } if (dst != src1) { - movdqu(dst, src1); + movaps(dst, src1); } if (CpuFeatures::IsSupported(SSE4_1)) { CpuFeatureScope sse_scope(this, SSE4_1); @@ -2429,7 +1977,7 @@ void TurboAssembler::Pinsrw(XMMRegister dst, XMMRegister src1, Operand src2, return; } else { if (dst != src1) { - movdqu(dst, src1); + movaps(dst, src1); } pinsrw(dst, src2, imm8); return; @@ -2446,17 +1994,6 @@ void TurboAssembler::Vbroadcastss(XMMRegister dst, Operand src) { shufps(dst, dst, static_cast<byte>(0)); } -void TurboAssembler::Extractps(Operand dst, XMMRegister src, uint8_t imm8) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vextractps(dst, src, imm8); - } - - DCHECK(CpuFeatures::IsSupported(SSE4_1)); - CpuFeatureScope avx_scope(this, SSE4_1); - extractps(dst, src, imm8); -} - void TurboAssembler::Shufps(XMMRegister dst, XMMRegister src1, XMMRegister src2, uint8_t imm8) { if (CpuFeatures::IsSupported(AVX)) { @@ -2732,6 +2269,12 @@ void TurboAssembler::CallBuiltin(int builtin_index) { call(entry, RelocInfo::OFF_HEAP_TARGET); } +Operand TurboAssembler::EntryFromBuiltinIndexAsOperand( + Builtins::Name builtin_index) { + return Operand(kRootRegister, + IsolateData::builtin_entry_slot_offset(builtin_index)); +} + void TurboAssembler::LoadCodeObjectEntry(Register destination, Register code_object) { // Code objects are called differently depending on whether we are generating diff --git a/deps/v8/src/codegen/ia32/macro-assembler-ia32.h b/deps/v8/src/codegen/ia32/macro-assembler-ia32.h index 29bb8ca2a0..4c5c3ade02 100644 --- a/deps/v8/src/codegen/ia32/macro-assembler-ia32.h +++ b/deps/v8/src/codegen/ia32/macro-assembler-ia32.h @@ -23,6 +23,7 @@ #include "src/codegen/label.h" #include "src/codegen/reglist.h" #include "src/codegen/reloc-info.h" +#include "src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.h" #include "src/codegen/turbo-assembler.h" #include "src/common/globals.h" #include "src/execution/frames.h" @@ -70,9 +71,9 @@ class StackArgumentsAccessor { DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor); }; -class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { +class V8_EXPORT_PRIVATE TurboAssembler : public SharedTurboAssembler { public: - using TurboAssemblerBase::TurboAssemblerBase; + using SharedTurboAssembler::SharedTurboAssembler; void CheckPageFlag(Register object, Register scratch, int mask, Condition cc, Label* condition_met, @@ -125,6 +126,7 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void Move(Register dst, Smi src) { Move(dst, Immediate(src)); } void Move(Register dst, Handle<HeapObject> src); void Move(Register dst, Register src); + void Move(Register dst, Operand src); void Move(Operand dst, const Immediate& src); // Move an immediate into an XMM register. @@ -133,7 +135,10 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void Move(XMMRegister dst, float src) { Move(dst, bit_cast<uint32_t>(src)); } void Move(XMMRegister dst, double src) { Move(dst, bit_cast<uint64_t>(src)); } + Operand EntryFromBuiltinIndexAsOperand(Builtins::Name builtin_index); + void Call(Register reg) { call(reg); } + void Call(Operand op) { call(op); } void Call(Label* target) { call(target); } void Call(Handle<Code> code_object, RelocInfo::Mode rmode); @@ -189,6 +194,10 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { } void SmiUntag(Register reg) { sar(reg, kSmiTagSize); } + void SmiUntag(Register output, Register value) { + mov(output, value); + SmiUntag(output); + } // Removes current frame and its arguments from the stack preserving the // arguments and a return address pushed to the stack for the next call. Both @@ -243,6 +252,13 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void PushReturnAddressFrom(Register src) { push(src); } void PopReturnAddressTo(Register dst) { pop(dst); } + void PushReturnAddressFrom(XMMRegister src, Register scratch) { + Push(src, scratch); + } + void PopReturnAddressTo(XMMRegister dst, Register scratch) { + Pop(dst, scratch); + } + void Ret(); // Root register utility functions. @@ -285,72 +301,6 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { // may be bigger than 2^16 - 1. Requires a scratch register. void Ret(int bytes_dropped, Register scratch); - // Three-operand cmpeqps that moves src1 to dst if AVX is not supported. - void Cmpeqps(XMMRegister dst, XMMRegister src1, XMMRegister src2); - - void Pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) { - Pshufhw(dst, Operand(src), shuffle); - } - void Pshufhw(XMMRegister dst, Operand src, uint8_t shuffle); - void Pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) { - Pshuflw(dst, Operand(src), shuffle); - } - void Pshuflw(XMMRegister dst, Operand src, uint8_t shuffle); - void Pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) { - Pshufd(dst, Operand(src), shuffle); - } - void Pshufd(XMMRegister dst, Operand src, uint8_t shuffle); - void Psraw(XMMRegister dst, uint8_t shift); - void Psrlw(XMMRegister dst, uint8_t shift); - void Psrlq(XMMRegister dst, uint8_t shift); - -// SSE/SSE2 instructions with AVX version. -#define AVX_OP2_WITH_TYPE(macro_name, name, dst_type, src_type) \ - void macro_name(dst_type dst, src_type src) { \ - if (CpuFeatures::IsSupported(AVX)) { \ - CpuFeatureScope scope(this, AVX); \ - v##name(dst, src); \ - } else { \ - name(dst, src); \ - } \ - } - - AVX_OP2_WITH_TYPE(Movss, movss, Operand, XMMRegister) - AVX_OP2_WITH_TYPE(Movss, movss, XMMRegister, Operand) - AVX_OP2_WITH_TYPE(Movsd, movsd, Operand, XMMRegister) - AVX_OP2_WITH_TYPE(Movsd, movsd, XMMRegister, Operand) - AVX_OP2_WITH_TYPE(Rcpps, rcpps, XMMRegister, const Operand&) - AVX_OP2_WITH_TYPE(Rsqrtps, rsqrtps, XMMRegister, const Operand&) - AVX_OP2_WITH_TYPE(Movdqu, movdqu, XMMRegister, Operand) - AVX_OP2_WITH_TYPE(Movdqu, movdqu, Operand, XMMRegister) - AVX_OP2_WITH_TYPE(Movd, movd, XMMRegister, Register) - AVX_OP2_WITH_TYPE(Movd, movd, XMMRegister, Operand) - AVX_OP2_WITH_TYPE(Movd, movd, Register, XMMRegister) - AVX_OP2_WITH_TYPE(Movd, movd, Operand, XMMRegister) - AVX_OP2_WITH_TYPE(Cvtdq2ps, cvtdq2ps, XMMRegister, Operand) - AVX_OP2_WITH_TYPE(Cvtdq2ps, cvtdq2ps, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Cvtdq2pd, cvtdq2pd, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Cvtps2pd, cvtps2pd, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Cvtpd2ps, cvtpd2ps, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Cvttps2dq, cvttps2dq, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Sqrtps, sqrtps, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Sqrtpd, sqrtpd, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Sqrtpd, sqrtpd, XMMRegister, const Operand&) - AVX_OP2_WITH_TYPE(Movaps, movaps, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Movups, movups, XMMRegister, Operand) - AVX_OP2_WITH_TYPE(Movups, movups, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Movups, movups, Operand, XMMRegister) - AVX_OP2_WITH_TYPE(Movapd, movapd, XMMRegister, XMMRegister) - AVX_OP2_WITH_TYPE(Movapd, movapd, XMMRegister, const Operand&) - AVX_OP2_WITH_TYPE(Movupd, movupd, XMMRegister, const Operand&) - AVX_OP2_WITH_TYPE(Pmovmskb, pmovmskb, Register, XMMRegister) - AVX_OP2_WITH_TYPE(Movmskpd, movmskpd, Register, XMMRegister) - AVX_OP2_WITH_TYPE(Movmskps, movmskps, Register, XMMRegister) - AVX_OP2_WITH_TYPE(Movlps, movlps, Operand, XMMRegister) - AVX_OP2_WITH_TYPE(Movhps, movlps, Operand, XMMRegister) - -#undef AVX_OP2_WITH_TYPE - // Only use these macros when non-destructive source of AVX version is not // needed. #define AVX_OP3_WITH_TYPE(macro_name, name, dst_type, src_type) \ @@ -384,7 +334,6 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP3_XO(Punpcklqdq, punpcklqdq) AVX_OP3_XO(Pxor, pxor) AVX_OP3_XO(Andps, andps) - AVX_OP3_XO(Andnps, andnps) AVX_OP3_XO(Andpd, andpd) AVX_OP3_XO(Xorps, xorps) AVX_OP3_XO(Xorpd, xorpd) @@ -393,12 +342,36 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP3_XO(Orps, orps) AVX_OP3_XO(Orpd, orpd) AVX_OP3_XO(Andnpd, andnpd) - AVX_OP3_XO(Pmullw, pmullw) AVX_OP3_WITH_TYPE(Movhlps, movhlps, XMMRegister, XMMRegister) + AVX_OP3_WITH_TYPE(Psraw, psraw, XMMRegister, uint8_t) + AVX_OP3_WITH_TYPE(Psrlq, psrlq, XMMRegister, uint8_t) #undef AVX_OP3_XO #undef AVX_OP3_WITH_TYPE +// Same as AVX_OP3_WITH_TYPE but supports a CpuFeatureScope +#define AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, dst_type, src_type, \ + sse_scope) \ + void macro_name(dst_type dst, src_type src) { \ + if (CpuFeatures::IsSupported(AVX)) { \ + CpuFeatureScope scope(this, AVX); \ + v##name(dst, dst, src); \ + } else if (CpuFeatures::IsSupported(sse_scope)) { \ + CpuFeatureScope scope(this, sse_scope); \ + name(dst, src); \ + } \ + } +#define AVX_OP2_XO(macro_name, name, sse_scope) \ + AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, \ + sse_scope) \ + AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, sse_scope) + AVX_OP2_XO(Psignb, psignb, SSSE3) + AVX_OP2_XO(Psignw, psignw, SSSE3) + AVX_OP2_XO(Psignd, psignd, SSSE3) + AVX_OP2_XO(Pcmpeqq, pcmpeqq, SSE4_1) +#undef AVX_OP2_XO +#undef AVX_OP2_WITH_TYPE_SCOPE + // Only use this macro when dst and src1 is the same in SSE case. #define AVX_PACKED_OP3_WITH_TYPE(macro_name, name, dst_type, src_type) \ void macro_name(dst_type dst, dst_type src1, src_type src2) { \ @@ -415,6 +388,7 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_PACKED_OP3_WITH_TYPE(macro_name, name, XMMRegister, Operand) AVX_PACKED_OP3(Unpcklps, unpcklps) + AVX_PACKED_OP3(Andnps, andnps) AVX_PACKED_OP3(Addps, addps) AVX_PACKED_OP3(Addpd, addpd) AVX_PACKED_OP3(Subps, subps) @@ -442,7 +416,6 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_PACKED_OP3(Psrlq, psrlq) AVX_PACKED_OP3(Psraw, psraw) AVX_PACKED_OP3(Psrad, psrad) - AVX_PACKED_OP3(Pmaddwd, pmaddwd) AVX_PACKED_OP3(Paddd, paddd) AVX_PACKED_OP3(Paddq, paddq) AVX_PACKED_OP3(Psubd, psubd) @@ -489,58 +462,12 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { name(dst, src2); \ } \ } + AVX_OP3_WITH_MOVE(Cmpeqps, cmpeqps, XMMRegister, XMMRegister) AVX_OP3_WITH_MOVE(Movlps, movlps, XMMRegister, Operand) AVX_OP3_WITH_MOVE(Movhps, movhps, XMMRegister, Operand) + AVX_OP3_WITH_MOVE(Pmaddwd, pmaddwd, XMMRegister, Operand) #undef AVX_OP3_WITH_MOVE -// Non-SSE2 instructions. -#define AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, dst_type, src_type, \ - sse_scope) \ - void macro_name(dst_type dst, src_type src) { \ - if (CpuFeatures::IsSupported(AVX)) { \ - CpuFeatureScope scope(this, AVX); \ - v##name(dst, src); \ - return; \ - } \ - if (CpuFeatures::IsSupported(sse_scope)) { \ - CpuFeatureScope scope(this, sse_scope); \ - name(dst, src); \ - return; \ - } \ - UNREACHABLE(); \ - } -#define AVX_OP2_XO_SSE3(macro_name, name) \ - AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, SSE3) \ - AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, SSE3) - AVX_OP2_XO_SSE3(Movddup, movddup) - AVX_OP2_WITH_TYPE_SCOPE(Movshdup, movshdup, XMMRegister, XMMRegister, SSE3) - -#undef AVX_OP2_XO_SSE3 - -#define AVX_OP2_XO_SSSE3(macro_name, name) \ - AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, SSSE3) \ - AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, SSSE3) - AVX_OP2_XO_SSSE3(Pabsb, pabsb) - AVX_OP2_XO_SSSE3(Pabsw, pabsw) - AVX_OP2_XO_SSSE3(Pabsd, pabsd) - -#undef AVX_OP2_XO_SSE3 - -#define AVX_OP2_XO_SSE4(macro_name, name) \ - AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, SSE4_1) \ - AVX_OP2_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, SSE4_1) - - AVX_OP2_XO_SSE4(Ptest, ptest) - AVX_OP2_XO_SSE4(Pmovsxbw, pmovsxbw) - AVX_OP2_XO_SSE4(Pmovsxwd, pmovsxwd) - AVX_OP2_XO_SSE4(Pmovsxdq, pmovsxdq) - AVX_OP2_XO_SSE4(Pmovzxbw, pmovzxbw) - AVX_OP2_XO_SSE4(Pmovzxwd, pmovzxwd) - AVX_OP2_XO_SSE4(Pmovzxdq, pmovzxdq) - -#undef AVX_OP2_WITH_TYPE_SCOPE -#undef AVX_OP2_XO_SSE4 - #define AVX_OP3_WITH_TYPE_SCOPE(macro_name, name, dst_type, src_type, \ sse_scope) \ void macro_name(dst_type dst, dst_type src1, src_type src2) { \ @@ -561,20 +488,23 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP3_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, XMMRegister, SSE4_1) \ AVX_OP3_WITH_TYPE_SCOPE(macro_name, name, XMMRegister, Operand, SSE4_1) + AVX_OP3_WITH_TYPE_SCOPE(Haddps, haddps, XMMRegister, Operand, SSE3) AVX_OP3_XO_SSE4(Pmaxsd, pmaxsd) AVX_OP3_XO_SSE4(Pminsb, pminsb) AVX_OP3_XO_SSE4(Pmaxsb, pmaxsb) - AVX_OP3_WITH_TYPE_SCOPE(Pmaddubsw, pmaddubsw, XMMRegister, XMMRegister, SSSE3) - AVX_OP3_WITH_TYPE_SCOPE(Pmaddubsw, pmaddubsw, XMMRegister, Operand, SSSE3) + AVX_OP3_XO_SSE4(Pcmpeqq, pcmpeqq) #undef AVX_OP3_XO_SSE4 #undef AVX_OP3_WITH_TYPE_SCOPE - void Haddps(XMMRegister dst, XMMRegister src1, Operand src2); - void Pcmpeqq(XMMRegister dst, Operand src); - void Pcmpeqq(XMMRegister dst, XMMRegister src) { Pcmpeqq(dst, Operand(src)); } - void Pcmpeqq(XMMRegister dst, XMMRegister src1, Operand src2); - void Pcmpeqq(XMMRegister dst, XMMRegister src1, XMMRegister src2); + // TODO(zhin): Remove after moving more definitions into SharedTurboAssembler. + void Movlps(Operand dst, XMMRegister src) { + SharedTurboAssembler::Movlps(dst, src); + } + void Movhps(Operand dst, XMMRegister src) { + SharedTurboAssembler::Movhps(dst, src); + } + void Pshufb(XMMRegister dst, XMMRegister src) { Pshufb(dst, dst, src); } void Pshufb(XMMRegister dst, Operand src) { Pshufb(dst, dst, src); } // Handles SSE and AVX. On SSE, moves src to dst if they are not equal. @@ -588,22 +518,11 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { } void Pblendw(XMMRegister dst, Operand src, uint8_t imm8); - void Psignb(XMMRegister dst, XMMRegister src) { Psignb(dst, Operand(src)); } - void Psignb(XMMRegister dst, Operand src); - void Psignw(XMMRegister dst, XMMRegister src) { Psignw(dst, Operand(src)); } - void Psignw(XMMRegister dst, Operand src); - void Psignd(XMMRegister dst, XMMRegister src) { Psignd(dst, Operand(src)); } - void Psignd(XMMRegister dst, Operand src); - void Palignr(XMMRegister dst, XMMRegister src, uint8_t imm8) { Palignr(dst, Operand(src), imm8); } void Palignr(XMMRegister dst, Operand src, uint8_t imm8); - void Pextrb(Operand dst, XMMRegister src, uint8_t imm8); - void Pextrb(Register dst, XMMRegister src, uint8_t imm8); - void Pextrw(Operand dst, XMMRegister src, uint8_t imm8); - void Pextrw(Register dst, XMMRegister src, uint8_t imm8); void Pextrd(Register dst, XMMRegister src, uint8_t imm8); void Pinsrb(XMMRegister dst, Register src, int8_t imm8) { Pinsrb(dst, Operand(src), imm8); @@ -624,7 +543,6 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { // Moves src1 to dst if AVX is not supported. void Pinsrw(XMMRegister dst, XMMRegister src1, Operand src2, int8_t imm8); void Vbroadcastss(XMMRegister dst, Operand src); - void Extractps(Operand dst, XMMRegister src, uint8_t imm8); // Shufps that will mov src1 into dst if AVX is not supported. void Shufps(XMMRegister dst, XMMRegister src1, XMMRegister src2, @@ -656,37 +574,14 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { } void Cvttsd2ui(Register dst, Operand src, XMMRegister tmp); - void Roundps(XMMRegister dst, XMMRegister src, RoundingMode mode); - void Roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode); - // Handles SSE and AVX. On SSE, moves src to dst if they are not equal. void Pmulhrsw(XMMRegister dst, XMMRegister src1, XMMRegister src2); // These Wasm SIMD ops do not have direct lowerings on IA32. These // helpers are optimized to produce the fastest and smallest codegen. // Defined here to allow usage on both TurboFan and Liftoff. - void I64x2ExtMul(XMMRegister dst, XMMRegister src1, XMMRegister src2, - XMMRegister scratch, bool low, bool is_signed); - // Requires that dst == src1 if AVX is not supported. - void I32x4ExtMul(XMMRegister dst, XMMRegister src1, XMMRegister src2, - XMMRegister scratch, bool low, bool is_signed); - void I16x8ExtMul(XMMRegister dst, XMMRegister src1, XMMRegister src2, - XMMRegister scratch, bool low, bool is_signed); - // Requires dst == mask when AVX is not supported. - void S128Select(XMMRegister dst, XMMRegister mask, XMMRegister src1, - XMMRegister src2, XMMRegister scratch); - void I64x2SConvertI32x4High(XMMRegister dst, XMMRegister src); - void I64x2UConvertI32x4High(XMMRegister dst, XMMRegister src, - XMMRegister scratch); - void I32x4SConvertI16x8High(XMMRegister dst, XMMRegister src); - void I32x4UConvertI16x8High(XMMRegister dst, XMMRegister src, - XMMRegister scratch); - void I16x8SConvertI8x16High(XMMRegister dst, XMMRegister src); - void I16x8UConvertI8x16High(XMMRegister dst, XMMRegister src, - XMMRegister scratch); void I16x8Q15MulRSatS(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister scratch); - void S128Store32Lane(Operand dst, XMMRegister src, uint8_t laneidx); void I8x16Popcnt(XMMRegister dst, XMMRegister src, XMMRegister tmp1, XMMRegister tmp2, Register scratch); void F64x2ConvertLowI32x4U(XMMRegister dst, XMMRegister src, Register tmp); @@ -694,11 +589,6 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { XMMRegister scratch, Register tmp); void I32x4TruncSatF64x2UZero(XMMRegister dst, XMMRegister src, XMMRegister scratch, Register tmp); - void I64x2Abs(XMMRegister dst, XMMRegister src, XMMRegister scratch); - void I64x2GtS(XMMRegister dst, XMMRegister src0, XMMRegister src1, - XMMRegister scratch); - void I64x2GeS(XMMRegister dst, XMMRegister src0, XMMRegister src1, - XMMRegister scratch); void I16x8ExtAddPairwiseI8x16S(XMMRegister dst, XMMRegister src, XMMRegister tmp, Register scratch); void I16x8ExtAddPairwiseI8x16U(XMMRegister dst, XMMRegister src, @@ -707,12 +597,25 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { Register scratch); void I32x4ExtAddPairwiseI16x8U(XMMRegister dst, XMMRegister src, XMMRegister tmp); + void I8x16Swizzle(XMMRegister dst, XMMRegister src, XMMRegister mask, + XMMRegister scratch, Register tmp, bool omit_add = false); void Push(Register src) { push(src); } void Push(Operand src) { push(src); } void Push(Immediate value); void Push(Handle<HeapObject> handle) { push(Immediate(handle)); } void Push(Smi smi) { Push(Immediate(smi)); } + void Push(XMMRegister src, Register scratch) { + movd(scratch, src); + push(scratch); + } + + void Pop(Register dst) { pop(dst); } + void Pop(Operand dst) { pop(dst); } + void Pop(XMMRegister dst, Register scratch) { + pop(scratch); + movd(dst, scratch); + } void SaveRegisters(RegList registers); void RestoreRegisters(RegList registers); @@ -994,9 +897,6 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // from the stack, clobbering only the esp register. void Drop(int element_count); - void Pop(Register dst) { pop(dst); } - void Pop(Operand dst) { pop(dst); } - // --------------------------------------------------------------------------- // In-place weak references. void LoadWeakValue(Register in_out, Label* target_if_cleared); diff --git a/deps/v8/src/codegen/ia32/register-ia32.h b/deps/v8/src/codegen/ia32/register-ia32.h index df3117e8d0..5dc035d966 100644 --- a/deps/v8/src/codegen/ia32/register-ia32.h +++ b/deps/v8/src/codegen/ia32/register-ia32.h @@ -76,7 +76,12 @@ GENERAL_REGISTERS(DEFINE_REGISTER) #undef DEFINE_REGISTER constexpr Register no_reg = Register::no_reg(); -constexpr bool kPadArguments = false; +// Returns the number of padding slots needed for stack pointer alignment. +constexpr int ArgumentPaddingSlots(int argument_count) { + // No argument padding required. + return 0; +} + constexpr bool kSimpleFPAliasing = true; constexpr bool kSimdMaskRegisters = false; diff --git a/deps/v8/src/codegen/interface-descriptors.cc b/deps/v8/src/codegen/interface-descriptors.cc index cb686a1bfa..53b678580e 100644 --- a/deps/v8/src/codegen/interface-descriptors.cc +++ b/deps/v8/src/codegen/interface-descriptors.cc @@ -25,6 +25,9 @@ void CallInterfaceDescriptorData::InitializePlatformSpecific( // within the calling convention are disallowed. #ifdef DEBUG CHECK_NE(registers[i], kRootRegister); +#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE + CHECK_NE(registers[i], kPointerCageBaseRegister); +#endif // Check for duplicated registers. for (int j = i + 1; j < register_parameter_count; j++) { CHECK_NE(registers[i], registers[j]); @@ -331,11 +334,16 @@ void StoreTransitionDescriptor::InitializePlatformSpecific( void BaselineOutOfLinePrologueDescriptor::InitializePlatformSpecific( CallInterfaceDescriptorData* data) { // TODO(v8:11421): Implement on other platforms. -#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64 - Register registers[] = { - kContextRegister, kJSFunctionRegister, kJavaScriptCallArgCountRegister, - kInterpreterBytecodeArrayRegister, kJavaScriptCallNewTargetRegister}; - data->InitializePlatformSpecific(kParameterCount, registers); +#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64 || V8_TARGET_ARCH_IA32 || \ + V8_TARGET_ARCH_ARM + Register registers[] = {kContextRegister, + kJSFunctionRegister, + kJavaScriptCallArgCountRegister, + kJavaScriptCallExtraArg1Register, + kJavaScriptCallNewTargetRegister, + kInterpreterBytecodeArrayRegister}; + data->InitializePlatformSpecific(kParameterCount - kStackArgumentsCount, + registers); #else InitializePlatformUnimplemented(data, kParameterCount); #endif @@ -344,7 +352,8 @@ void BaselineOutOfLinePrologueDescriptor::InitializePlatformSpecific( void BaselineLeaveFrameDescriptor::InitializePlatformSpecific( CallInterfaceDescriptorData* data) { // TODO(v8:11421): Implement on other platforms. -#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64 +#if V8_TARGET_ARCH_IA32 || V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64 || \ + V8_TARGET_ARCH_ARM Register registers[] = {ParamsSizeRegister(), WeightRegister()}; data->InitializePlatformSpecific(kParameterCount, registers); #else @@ -609,5 +618,15 @@ void ForInPrepareDescriptor::InitializePlatformSpecific( DefaultInitializePlatformSpecific(data, kParameterCount); } +void SuspendGeneratorBaselineDescriptor::InitializePlatformSpecific( + CallInterfaceDescriptorData* data) { + DefaultInitializePlatformSpecific(data, kParameterCount); +} + +void ResumeGeneratorBaselineDescriptor::InitializePlatformSpecific( + CallInterfaceDescriptorData* data) { + DefaultInitializePlatformSpecific(data, kParameterCount); +} + } // namespace internal } // namespace v8 diff --git a/deps/v8/src/codegen/interface-descriptors.h b/deps/v8/src/codegen/interface-descriptors.h index d9ae65f5c6..8d03907efc 100644 --- a/deps/v8/src/codegen/interface-descriptors.h +++ b/deps/v8/src/codegen/interface-descriptors.h @@ -97,6 +97,8 @@ namespace internal { V(NoContext) \ V(RecordWrite) \ V(ResumeGenerator) \ + V(SuspendGeneratorBaseline) \ + V(ResumeGeneratorBaseline) \ V(RunMicrotasks) \ V(RunMicrotasksEntry) \ V(SingleParameterOnStack) \ @@ -1466,16 +1468,26 @@ class V8_EXPORT_PRIVATE TailCallOptimizedCodeSlotDescriptor class BaselineOutOfLinePrologueDescriptor : public CallInterfaceDescriptor { public: DEFINE_PARAMETERS_NO_CONTEXT(kCalleeContext, kClosure, - kJavaScriptCallArgCount, - kInterpreterBytecodeArray, - kJavaScriptCallNewTarget) + kJavaScriptCallArgCount, kStackFrameSize, + kJavaScriptCallNewTarget, + kInterpreterBytecodeArray) DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kCalleeContext MachineType::AnyTagged(), // kClosure MachineType::Int32(), // kJavaScriptCallArgCount - MachineType::AnyTagged(), // kInterpreterBytecodeArray - MachineType::AnyTagged()) // kJavaScriptCallNewTarget + MachineType::Int32(), // kStackFrameSize + MachineType::AnyTagged(), // kJavaScriptCallNewTarget + MachineType::AnyTagged()) // kInterpreterBytecodeArray DECLARE_DESCRIPTOR(BaselineOutOfLinePrologueDescriptor, CallInterfaceDescriptor) + +#if V8_TARGET_ARCH_IA32 + static const bool kPassLastArgsOnStack = true; +#else + static const bool kPassLastArgsOnStack = false; +#endif + + // Pass bytecode array through the stack. + static const int kStackArgumentsCount = kPassLastArgsOnStack ? 1 : 0; }; class BaselineLeaveFrameDescriptor : public CallInterfaceDescriptor { @@ -1577,6 +1589,31 @@ class ResumeGeneratorDescriptor final : public CallInterfaceDescriptor { DECLARE_DESCRIPTOR(ResumeGeneratorDescriptor, CallInterfaceDescriptor) }; +class ResumeGeneratorBaselineDescriptor final : public CallInterfaceDescriptor { + public: + DEFINE_PARAMETERS(kGeneratorObject, kRegisterCount) + DEFINE_RESULT_AND_PARAMETER_TYPES( + MachineType::TaggedSigned(), // return type + MachineType::AnyTagged(), // kGeneratorObject + MachineType::IntPtr(), // kRegisterCount + ) + DECLARE_DESCRIPTOR(ResumeGeneratorBaselineDescriptor, CallInterfaceDescriptor) +}; + +class SuspendGeneratorBaselineDescriptor final + : public CallInterfaceDescriptor { + public: + DEFINE_PARAMETERS(kGeneratorObject, kSuspendId, kBytecodeOffset, + kRegisterCount) + DEFINE_PARAMETER_TYPES(MachineType::AnyTagged(), // kGeneratorObject + MachineType::IntPtr(), // kSuspendId + MachineType::IntPtr(), // kBytecodeOffset + MachineType::IntPtr(), // kRegisterCount + ) + DECLARE_DESCRIPTOR(SuspendGeneratorBaselineDescriptor, + CallInterfaceDescriptor) +}; + class FrameDropperTrampolineDescriptor final : public CallInterfaceDescriptor { public: DEFINE_PARAMETERS(kRestartFp) diff --git a/deps/v8/src/codegen/machine-type.h b/deps/v8/src/codegen/machine-type.h index cd46fd1aef..ac21d3c3e6 100644 --- a/deps/v8/src/codegen/machine-type.h +++ b/deps/v8/src/codegen/machine-type.h @@ -17,6 +17,7 @@ namespace internal { enum class MachineRepresentation : uint8_t { kNone, kBit, + // Integral representations must be consecutive, in order of increasing order. kWord8, kWord16, kWord32, @@ -26,7 +27,7 @@ enum class MachineRepresentation : uint8_t { kTagged, // (uncompressed) Object (Smi or HeapObject) kCompressedPointer, // (compressed) HeapObject kCompressed, // (compressed) Object (Smi or HeapObject) - // FP representations must be last, and in order of increasing size. + // FP and SIMD representations must be last, and in order of increasing size. kFloat32, kFloat64, kSimd128, @@ -36,6 +37,22 @@ enum class MachineRepresentation : uint8_t { bool IsSubtype(MachineRepresentation rep1, MachineRepresentation rep2); +#define ASSERT_CONSECUTIVE(rep1, rep2) \ + static_assert(static_cast<uint8_t>(MachineRepresentation::k##rep1) + 1 == \ + static_cast<uint8_t>(MachineRepresentation::k##rep2), \ + #rep1 " and " #rep2 " must be consecutive."); + +ASSERT_CONSECUTIVE(Word8, Word16) +ASSERT_CONSECUTIVE(Word16, Word32) +ASSERT_CONSECUTIVE(Word32, Word64) +ASSERT_CONSECUTIVE(Float32, Float64) +ASSERT_CONSECUTIVE(Float64, Simd128) +#undef ASSERT_CONSECUTIVE + +static_assert(MachineRepresentation::kLastRepresentation == + MachineRepresentation::kSimd128, + "FP and SIMD representations must be last."); + static_assert(static_cast<int>(MachineRepresentation::kLastRepresentation) < kIntSize * kBitsPerByte, "Bit masks of MachineRepresentation should fit in an int"); @@ -255,6 +272,11 @@ V8_EXPORT_PRIVATE std::ostream& operator<<(std::ostream& os, std::ostream& operator<<(std::ostream& os, MachineSemantic type); V8_EXPORT_PRIVATE std::ostream& operator<<(std::ostream& os, MachineType type); +inline bool IsIntegral(MachineRepresentation rep) { + return rep >= MachineRepresentation::kWord8 && + rep <= MachineRepresentation::kWord64; +} + inline bool IsFloatingPoint(MachineRepresentation rep) { return rep >= MachineRepresentation::kFirstFPRepresentation; } diff --git a/deps/v8/src/codegen/mips/assembler-mips-inl.h b/deps/v8/src/codegen/mips/assembler-mips-inl.h index 8ffc46003f..d00da6efba 100644 --- a/deps/v8/src/codegen/mips/assembler-mips-inl.h +++ b/deps/v8/src/codegen/mips/assembler-mips-inl.h @@ -47,8 +47,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return IsSupported(FPU); } -bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(MIPS_SIMD); } - // ----------------------------------------------------------------------------- // Operand and MemOperand. diff --git a/deps/v8/src/codegen/mips/assembler-mips.cc b/deps/v8/src/codegen/mips/assembler-mips.cc index 9864c0f253..2ef08ae87c 100644 --- a/deps/v8/src/codegen/mips/assembler-mips.cc +++ b/deps/v8/src/codegen/mips/assembler-mips.cc @@ -67,6 +67,8 @@ static unsigned CpuFeaturesImpliedByCompiler() { return answer; } +bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(MIPS_SIMD); } + void CpuFeatures::ProbeImpl(bool cross_compile) { supported_ |= CpuFeaturesImpliedByCompiler(); diff --git a/deps/v8/src/codegen/mips/macro-assembler-mips.cc b/deps/v8/src/codegen/mips/macro-assembler-mips.cc index 060a4b748a..8bbdbca662 100644 --- a/deps/v8/src/codegen/mips/macro-assembler-mips.cc +++ b/deps/v8/src/codegen/mips/macro-assembler-mips.cc @@ -23,7 +23,10 @@ #include "src/runtime/runtime.h" #include "src/snapshot/embedded/embedded-data.h" #include "src/snapshot/snapshot.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY // Satisfy cpplint check, but don't include platform-specific header. It is // included recursively via macro-assembler.h. @@ -2741,8 +2744,13 @@ void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone, Subu(sp, sp, Operand(kDoubleSize)); // Put input on stack. Sdc1(double_input, MemOperand(sp, 0)); +#if V8_ENABLE_WEBASSEMBLY if (stub_mode == StubCallMode::kCallWasmRuntimeStub) { Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL); +#else + // For balance. + if (false) { +#endif // V8_ENABLE_WEBASSEMBLY } else { Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET); } @@ -4805,7 +4813,7 @@ void TurboAssembler::LoadMap(Register destination, Register object) { Lw(destination, FieldMemOperand(object, HeapObject::kMapOffset)); } -void MacroAssembler::LoadNativeContextSlot(int index, Register dst) { +void MacroAssembler::LoadNativeContextSlot(Register dst, int index) { LoadMap(dst, cp); Lw(dst, FieldMemOperand(dst, Map::kConstructorOrBackPointerOrNativeContextOffset)); diff --git a/deps/v8/src/codegen/mips/macro-assembler-mips.h b/deps/v8/src/codegen/mips/macro-assembler-mips.h index 1fe4c451f9..8d54e0b737 100644 --- a/deps/v8/src/codegen/mips/macro-assembler-mips.h +++ b/deps/v8/src/codegen/mips/macro-assembler-mips.h @@ -83,6 +83,13 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { } void LeaveFrame(StackFrame::Type type); + void AllocateStackSpace(Register bytes) { Subu(sp, sp, bytes); } + void AllocateStackSpace(int bytes) { + DCHECK_GE(bytes, 0); + if (bytes == 0) return; + Subu(sp, sp, Operand(bytes)); + } + // Generates function and stub prologue code. void StubPrologue(StackFrame::Type type); void Prologue(); @@ -998,10 +1005,10 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // Load the global proxy from the current context. void LoadGlobalProxy(Register dst) { - LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst); + LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX); } - void LoadNativeContextSlot(int index, Register dst); + void LoadNativeContextSlot(Register dst, int index); // ------------------------------------------------------------------------- // JavaScript invokes. diff --git a/deps/v8/src/codegen/mips/register-mips.h b/deps/v8/src/codegen/mips/register-mips.h index 2b5f454dd4..95164a86c1 100644 --- a/deps/v8/src/codegen/mips/register-mips.h +++ b/deps/v8/src/codegen/mips/register-mips.h @@ -203,7 +203,12 @@ int ToNumber(Register reg); Register ToRegister(int num); -constexpr bool kPadArguments = false; +// Returns the number of padding slots needed for stack pointer alignment. +constexpr int ArgumentPaddingSlots(int argument_count) { + // No argument padding required. + return 0; +} + constexpr bool kSimpleFPAliasing = true; constexpr bool kSimdMaskRegisters = false; diff --git a/deps/v8/src/codegen/mips64/assembler-mips64-inl.h b/deps/v8/src/codegen/mips64/assembler-mips64-inl.h index e5ac8d209c..2924b661f2 100644 --- a/deps/v8/src/codegen/mips64/assembler-mips64-inl.h +++ b/deps/v8/src/codegen/mips64/assembler-mips64-inl.h @@ -47,8 +47,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return IsSupported(FPU); } -bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(MIPS_SIMD); } - // ----------------------------------------------------------------------------- // Operand and MemOperand. diff --git a/deps/v8/src/codegen/mips64/assembler-mips64.cc b/deps/v8/src/codegen/mips64/assembler-mips64.cc index 11dd818922..7f7ebd2c73 100644 --- a/deps/v8/src/codegen/mips64/assembler-mips64.cc +++ b/deps/v8/src/codegen/mips64/assembler-mips64.cc @@ -67,6 +67,8 @@ static unsigned CpuFeaturesImpliedByCompiler() { return answer; } +bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(MIPS_SIMD); } + void CpuFeatures::ProbeImpl(bool cross_compile) { supported_ |= CpuFeaturesImpliedByCompiler(); diff --git a/deps/v8/src/codegen/mips64/macro-assembler-mips64.cc b/deps/v8/src/codegen/mips64/macro-assembler-mips64.cc index ec19fa5db5..29443a2e58 100644 --- a/deps/v8/src/codegen/mips64/macro-assembler-mips64.cc +++ b/deps/v8/src/codegen/mips64/macro-assembler-mips64.cc @@ -23,7 +23,10 @@ #include "src/runtime/runtime.h" #include "src/snapshot/embedded/embedded-data.h" #include "src/snapshot/snapshot.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY // Satisfy cpplint check, but don't include platform-specific header. It is // included recursively via macro-assembler.h. @@ -3379,8 +3382,13 @@ void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone, Dsubu(sp, sp, Operand(kDoubleSize)); // Put input on stack. Sdc1(double_input, MemOperand(sp, 0)); +#if V8_ENABLE_WEBASSEMBLY if (stub_mode == StubCallMode::kCallWasmRuntimeStub) { Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL); +#else + // For balance. + if (false) { +#endif // V8_ENABLE_WEBASSEMBLY } else { Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET); } @@ -5326,7 +5334,7 @@ void TurboAssembler::LoadMap(Register destination, Register object) { Ld(destination, FieldMemOperand(object, HeapObject::kMapOffset)); } -void MacroAssembler::LoadNativeContextSlot(int index, Register dst) { +void MacroAssembler::LoadNativeContextSlot(Register dst, int index) { LoadMap(dst, cp); Ld(dst, FieldMemOperand(dst, Map::kConstructorOrBackPointerOrNativeContextOffset)); diff --git a/deps/v8/src/codegen/mips64/macro-assembler-mips64.h b/deps/v8/src/codegen/mips64/macro-assembler-mips64.h index 721326ae96..756b594edb 100644 --- a/deps/v8/src/codegen/mips64/macro-assembler-mips64.h +++ b/deps/v8/src/codegen/mips64/macro-assembler-mips64.h @@ -103,6 +103,14 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { } void LeaveFrame(StackFrame::Type type); + void AllocateStackSpace(Register bytes) { Dsubu(sp, sp, bytes); } + + void AllocateStackSpace(int bytes) { + DCHECK_GE(bytes, 0); + if (bytes == 0) return; + Dsubu(sp, sp, Operand(bytes)); + } + // Generates function and stub prologue code. void StubPrologue(StackFrame::Type type); void Prologue(); @@ -1048,10 +1056,10 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // Load the global proxy from the current context. void LoadGlobalProxy(Register dst) { - LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst); + LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX); } - void LoadNativeContextSlot(int index, Register dst); + void LoadNativeContextSlot(Register dst, int index); // Load the initial map from the global function. The registers // function and map can be the same, function is then overwritten. diff --git a/deps/v8/src/codegen/mips64/register-mips64.h b/deps/v8/src/codegen/mips64/register-mips64.h index 8267d6b2ff..51b03aba1f 100644 --- a/deps/v8/src/codegen/mips64/register-mips64.h +++ b/deps/v8/src/codegen/mips64/register-mips64.h @@ -203,7 +203,12 @@ int ToNumber(Register reg); Register ToRegister(int num); -constexpr bool kPadArguments = false; +// Returns the number of padding slots needed for stack pointer alignment. +constexpr int ArgumentPaddingSlots(int argument_count) { + // No argument padding required. + return 0; +} + constexpr bool kSimpleFPAliasing = true; constexpr bool kSimdMaskRegisters = false; diff --git a/deps/v8/src/codegen/optimized-compilation-info.cc b/deps/v8/src/codegen/optimized-compilation-info.cc index 2dc30fb55d..f6fd5862fd 100644 --- a/deps/v8/src/codegen/optimized-compilation-info.cc +++ b/deps/v8/src/codegen/optimized-compilation-info.cc @@ -13,7 +13,10 @@ #include "src/objects/shared-function-info.h" #include "src/tracing/trace-event.h" #include "src/tracing/traced-value.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/function-compiler.h" +#endif // V8_ENABLE_WEBASSEMBLY namespace v8 { namespace internal { @@ -170,6 +173,7 @@ StackFrame::Type OptimizedCompilationInfo::GetOutputStackFrameType() const { case CodeKind::BYTECODE_HANDLER: case CodeKind::BUILTIN: return StackFrame::STUB; +#if V8_ENABLE_WEBASSEMBLY case CodeKind::WASM_FUNCTION: return StackFrame::WASM; case CodeKind::WASM_TO_CAPI_FUNCTION: @@ -180,6 +184,7 @@ StackFrame::Type OptimizedCompilationInfo::GetOutputStackFrameType() const { return StackFrame::WASM_TO_JS; case CodeKind::C_WASM_ENTRY: return StackFrame::C_WASM_ENTRY; +#endif // V8_ENABLE_WEBASSEMBLY default: UNIMPLEMENTED(); return StackFrame::NONE; @@ -191,6 +196,7 @@ void OptimizedCompilationInfo::SetCode(Handle<Code> code) { code_ = code; } +#if V8_ENABLE_WEBASSEMBLY void OptimizedCompilationInfo::SetWasmCompilationResult( std::unique_ptr<wasm::WasmCompilationResult> wasm_compilation_result) { wasm_compilation_result_ = std::move(wasm_compilation_result); @@ -200,6 +206,7 @@ std::unique_ptr<wasm::WasmCompilationResult> OptimizedCompilationInfo::ReleaseWasmCompilationResult() { return std::move(wasm_compilation_result_); } +#endif // V8_ENABLE_WEBASSEMBLY bool OptimizedCompilationInfo::has_context() const { return !closure().is_null(); diff --git a/deps/v8/src/codegen/optimized-compilation-info.h b/deps/v8/src/codegen/optimized-compilation-info.h index b72526ab61..20386cbbee 100644 --- a/deps/v8/src/codegen/optimized-compilation-info.h +++ b/deps/v8/src/codegen/optimized-compilation-info.h @@ -142,8 +142,10 @@ class V8_EXPORT_PRIVATE OptimizedCompilationInfo final { void SetCode(Handle<Code> code); +#if V8_ENABLE_WEBASSEMBLY void SetWasmCompilationResult(std::unique_ptr<wasm::WasmCompilationResult>); std::unique_ptr<wasm::WasmCompilationResult> ReleaseWasmCompilationResult(); +#endif // V8_ENABLE_WEBASSEMBLY bool has_context() const; Context context() const; @@ -162,7 +164,9 @@ class V8_EXPORT_PRIVATE OptimizedCompilationInfo final { return code_kind() == CodeKind::NATIVE_CONTEXT_INDEPENDENT; } bool IsTurboprop() const { return code_kind() == CodeKind::TURBOPROP; } +#if V8_ENABLE_WEBASSEMBLY bool IsWasm() const { return code_kind() == CodeKind::WASM_FUNCTION; } +#endif // V8_ENABLE_WEBASSEMBLY void SetOptimizingForOsr(BytecodeOffset osr_offset, JavaScriptFrame* osr_frame) { @@ -283,8 +287,10 @@ class V8_EXPORT_PRIVATE OptimizedCompilationInfo final { // Basic block profiling support. BasicBlockProfilerData* profiler_data_ = nullptr; +#if V8_ENABLE_WEBASSEMBLY // The WebAssembly compilation result, not published in the NativeModule yet. std::unique_ptr<wasm::WasmCompilationResult> wasm_compilation_result_; +#endif // V8_ENABLE_WEBASSEMBLY // Entry point when compiling for OSR, {BytecodeOffset::None} otherwise. BytecodeOffset osr_offset_ = BytecodeOffset::None(); diff --git a/deps/v8/src/codegen/ppc/assembler-ppc-inl.h b/deps/v8/src/codegen/ppc/assembler-ppc-inl.h index a638ba9759..d8cd524451 100644 --- a/deps/v8/src/codegen/ppc/assembler-ppc-inl.h +++ b/deps/v8/src/codegen/ppc/assembler-ppc-inl.h @@ -48,8 +48,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return true; } -bool CpuFeatures::SupportsWasmSimd128() { return false; } - void RelocInfo::apply(intptr_t delta) { // absolute code pointer inside code object moves with the code object. if (IsInternalReference(rmode_)) { diff --git a/deps/v8/src/codegen/ppc/assembler-ppc.cc b/deps/v8/src/codegen/ppc/assembler-ppc.cc index 02e50e5fa3..7da9484cce 100644 --- a/deps/v8/src/codegen/ppc/assembler-ppc.cc +++ b/deps/v8/src/codegen/ppc/assembler-ppc.cc @@ -54,6 +54,10 @@ static unsigned CpuFeaturesImpliedByCompiler() { return answer; } +bool CpuFeatures::SupportsWasmSimd128() { + return CpuFeatures::IsSupported(SIMD); +} + void CpuFeatures::ProbeImpl(bool cross_compile) { supported_ |= CpuFeaturesImpliedByCompiler(); icache_line_size_ = 128; @@ -67,33 +71,42 @@ void CpuFeatures::ProbeImpl(bool cross_compile) { #ifndef USE_SIMULATOR // Probe for additional features at runtime. base::CPU cpu; - if (cpu.part() == base::CPU::PPC_POWER9) { + if (cpu.part() == base::CPU::kPPCPower9 || + cpu.part() == base::CPU::kPPCPower10) { supported_ |= (1u << MODULO); } #if V8_TARGET_ARCH_PPC64 - if (cpu.part() == base::CPU::PPC_POWER8 || - cpu.part() == base::CPU::PPC_POWER9) { + if (cpu.part() == base::CPU::kPPCPower8 || + cpu.part() == base::CPU::kPPCPower9 || + cpu.part() == base::CPU::kPPCPower10) { supported_ |= (1u << FPR_GPR_MOV); } + // V8 PPC Simd implementations need P9 at a minimum. + if (cpu.part() == base::CPU::kPPCPower9 || + cpu.part() == base::CPU::kPPCPower10) { + supported_ |= (1u << SIMD); + } #endif - if (cpu.part() == base::CPU::PPC_POWER6 || - cpu.part() == base::CPU::PPC_POWER7 || - cpu.part() == base::CPU::PPC_POWER8 || - cpu.part() == base::CPU::PPC_POWER9) { + if (cpu.part() == base::CPU::kPPCPower6 || + cpu.part() == base::CPU::kPPCPower7 || + cpu.part() == base::CPU::kPPCPower8 || + cpu.part() == base::CPU::kPPCPower9 || + cpu.part() == base::CPU::kPPCPower10) { supported_ |= (1u << LWSYNC); } - if (cpu.part() == base::CPU::PPC_POWER7 || - cpu.part() == base::CPU::PPC_POWER8 || - cpu.part() == base::CPU::PPC_POWER9) { + if (cpu.part() == base::CPU::kPPCPower7 || + cpu.part() == base::CPU::kPPCPower8 || + cpu.part() == base::CPU::kPPCPower9 || + cpu.part() == base::CPU::kPPCPower10) { supported_ |= (1u << ISELECT); supported_ |= (1u << VSX); } #if V8_OS_LINUX - if (!(cpu.part() == base::CPU::PPC_G5 || cpu.part() == base::CPU::PPC_G4)) { + if (!(cpu.part() == base::CPU::kPPCG5 || cpu.part() == base::CPU::kPPCG4)) { // Assume support supported_ |= (1u << FPU); } - if (cpu.icache_line_size() != base::CPU::UNKNOWN_CACHE_LINE_SIZE) { + if (cpu.icache_line_size() != base::CPU::kUnknownCacheLineSize) { icache_line_size_ = cpu.icache_line_size(); } #elif V8_OS_AIX @@ -106,6 +119,7 @@ void CpuFeatures::ProbeImpl(bool cross_compile) { supported_ |= (1u << ISELECT); supported_ |= (1u << VSX); supported_ |= (1u << MODULO); + supported_ |= (1u << SIMD); #if V8_TARGET_ARCH_PPC64 supported_ |= (1u << FPR_GPR_MOV); #endif @@ -1475,7 +1489,7 @@ void Assembler::mcrfs(CRegister cr, FPSCRBit bit) { void Assembler::mfcr(Register dst) { emit(EXT2 | MFCR | dst.code() * B21); } -void Assembler::mtcrf(unsigned char FXM, Register src) { +void Assembler::mtcrf(Register src, uint8_t FXM) { emit(MTCRF | src.code() * B21 | FXM * B12); } #if V8_TARGET_ARCH_PPC64 @@ -1870,6 +1884,12 @@ void Assembler::xxspltib(const Simd128Register rt, const Operand& imm) { emit(XXSPLTIB | rt.code() * B21 | imm.immediate() * B11 | TX); } +void Assembler::xxbrq(const Simd128Register rt, const Simd128Register rb) { + int BX = 1; + int TX = 1; + emit(XXBRQ | rt.code() * B21 | 31 * B16 | rb.code() * B11 | BX * B1 | TX); +} + // Pseudo instructions. void Assembler::nop(int type) { Register reg = r0; diff --git a/deps/v8/src/codegen/ppc/assembler-ppc.h b/deps/v8/src/codegen/ppc/assembler-ppc.h index a7c729c43b..d5b37fe59f 100644 --- a/deps/v8/src/codegen/ppc/assembler-ppc.h +++ b/deps/v8/src/codegen/ppc/assembler-ppc.h @@ -195,6 +195,12 @@ class Assembler : public AssemblerBase { void MaybeEmitOutOfLineConstantPool() { EmitConstantPool(); } + inline void CheckTrampolinePoolQuick(int extra_space = 0) { + if (pc_offset() >= next_trampoline_check_ - extra_space) { + CheckTrampolinePool(); + } + } + // Label operations & relative jumps (PPUM Appendix D) // // Takes a branch opcode (cc) and a label (L) and generates @@ -922,7 +928,7 @@ class Assembler : public AssemblerBase { void mtxer(Register src); void mcrfs(CRegister cr, FPSCRBit bit); void mfcr(Register dst); - void mtcrf(unsigned char FXM, Register src); + void mtcrf(Register src, uint8_t FXM); #if V8_TARGET_ARCH_PPC64 void mffprd(Register dst, DoubleRegister src); void mffprwz(Register dst, DoubleRegister src); @@ -1032,6 +1038,7 @@ class Assembler : public AssemblerBase { void stxsiwx(const Simd128Register rs, const MemOperand& src); void stxvd(const Simd128Register rt, const MemOperand& src); void xxspltib(const Simd128Register rt, const Operand& imm); + void xxbrq(const Simd128Register rt, const Simd128Register rb); // Pseudo instructions @@ -1331,12 +1338,6 @@ class Assembler : public AssemblerBase { } inline void UntrackBranch(); - void CheckTrampolinePoolQuick() { - if (pc_offset() >= next_trampoline_check_) { - CheckTrampolinePool(); - } - } - // Instruction generation void a_form(Instr instr, DoubleRegister frt, DoubleRegister fra, DoubleRegister frb, RCBit r); diff --git a/deps/v8/src/codegen/ppc/constants-ppc.h b/deps/v8/src/codegen/ppc/constants-ppc.h index 31bbb48044..5b37a2ee11 100644 --- a/deps/v8/src/codegen/ppc/constants-ppc.h +++ b/deps/v8/src/codegen/ppc/constants-ppc.h @@ -359,49 +359,64 @@ using Instr = uint32_t; /* Decimal Floating Test Data Group Quad */ \ V(dtstdgq, DTSTDGQ, 0xFC0001C4) -#define PPC_XX2_OPCODE_A_FORM_LIST(V) \ - /* VSX Vector Absolute Value Double-Precision */ \ - V(xvabsdp, XVABSDP, 0xF0000764) \ - /* VSX Vector Negate Double-Precision */ \ - V(xvnegdp, XVNEGDP, 0xF00007E4) \ - /* VSX Vector Square Root Double-Precision */ \ - V(xvsqrtdp, XVSQRTDP, 0xF000032C) \ - /* VSX Vector Absolute Value Single-Precision */ \ - V(xvabssp, XVABSSP, 0xF0000664) \ - /* VSX Vector Negate Single-Precision */ \ - V(xvnegsp, XVNEGSP, 0xF00006E4) \ - /* VSX Vector Reciprocal Estimate Single-Precision */ \ - V(xvresp, XVRESP, 0xF0000268) \ - /* VSX Vector Reciprocal Square Root Estimate Single-Precision */ \ - V(xvrsqrtesp, XVRSQRTESP, 0xF0000228) \ - /* VSX Vector Square Root Single-Precision */ \ - V(xvsqrtsp, XVSQRTSP, 0xF000022C) \ - /* VSX Vector Convert Single-Precision to Signed Fixed-Point Word */ \ - /* Saturate */ \ - V(xvcvspsxws, XVCVSPSXWS, 0xF0000260) \ - /* VSX Vector Convert Single-Precision to Unsigned Fixed-Point Word */ \ - /* Saturate */ \ - V(xvcvspuxws, XVCVSPUXWS, 0xF0000220) \ - /* VSX Vector Convert Signed Fixed-Point Word to Single-Precision */ \ - V(xvcvsxwsp, XVCVSXWSP, 0xF00002E0) \ - /* VSX Vector Convert Unsigned Fixed-Point Word to Single-Precision */ \ - V(xvcvuxwsp, XVCVUXWSP, 0xF00002A0) \ - /* VSX Vector Round to Double-Precision Integer toward +Infinity */ \ - V(xvrdpip, XVRDPIP, 0xF00003A4) \ - /* VSX Vector Round to Double-Precision Integer toward -Infinity */ \ - V(xvrdpim, XVRDPIM, 0xF00003E4) \ - /* VSX Vector Round to Double-Precision Integer toward Zero */ \ - V(xvrdpiz, XVRDPIZ, 0xF0000364) \ - /* VSX Vector Round to Double-Precision Integer */ \ - V(xvrdpi, XVRDPI, 0xF0000324) \ - /* VSX Vector Round to Single-Precision Integer toward +Infinity */ \ - V(xvrspip, XVRSPIP, 0xF00002A4) \ - /* VSX Vector Round to Single-Precision Integer toward -Infinity */ \ - V(xvrspim, XVRSPIM, 0xF00002E4) \ - /* VSX Vector Round to Single-Precision Integer toward Zero */ \ - V(xvrspiz, XVRSPIZ, 0xF0000264) \ - /* VSX Vector Round to Single-Precision Integer */ \ - V(xvrspi, XVRSPI, 0xF0000224) +#define PPC_XX2_OPCODE_A_FORM_LIST(V) \ + /* VSX Vector Absolute Value Double-Precision */ \ + V(xvabsdp, XVABSDP, 0xF0000764) \ + /* VSX Vector Negate Double-Precision */ \ + V(xvnegdp, XVNEGDP, 0xF00007E4) \ + /* VSX Vector Square Root Double-Precision */ \ + V(xvsqrtdp, XVSQRTDP, 0xF000032C) \ + /* VSX Vector Absolute Value Single-Precision */ \ + V(xvabssp, XVABSSP, 0xF0000664) \ + /* VSX Vector Negate Single-Precision */ \ + V(xvnegsp, XVNEGSP, 0xF00006E4) \ + /* VSX Vector Reciprocal Estimate Single-Precision */ \ + V(xvresp, XVRESP, 0xF0000268) \ + /* VSX Vector Reciprocal Square Root Estimate Single-Precision */ \ + V(xvrsqrtesp, XVRSQRTESP, 0xF0000228) \ + /* VSX Vector Square Root Single-Precision */ \ + V(xvsqrtsp, XVSQRTSP, 0xF000022C) \ + /* VSX Vector Convert Single-Precision to Signed Fixed-Point Word */ \ + /* Saturate */ \ + V(xvcvspsxws, XVCVSPSXWS, 0xF0000260) \ + /* VSX Vector Convert Single-Precision to Unsigned Fixed-Point Word */ \ + /* Saturate */ \ + V(xvcvspuxws, XVCVSPUXWS, 0xF0000220) \ + /* VSX Vector Convert Signed Fixed-Point Word to Single-Precision */ \ + V(xvcvsxwsp, XVCVSXWSP, 0xF00002E0) \ + /* VSX Vector Convert Unsigned Fixed-Point Word to Single-Precision */ \ + V(xvcvuxwsp, XVCVUXWSP, 0xF00002A0) \ + /* VSX Vector Round to Double-Precision Integer toward +Infinity */ \ + V(xvrdpip, XVRDPIP, 0xF00003A4) \ + /* VSX Vector Round to Double-Precision Integer toward -Infinity */ \ + V(xvrdpim, XVRDPIM, 0xF00003E4) \ + /* VSX Vector Round to Double-Precision Integer toward Zero */ \ + V(xvrdpiz, XVRDPIZ, 0xF0000364) \ + /* VSX Vector Round to Double-Precision Integer */ \ + V(xvrdpi, XVRDPI, 0xF0000324) \ + /* VSX Vector Round to Single-Precision Integer toward +Infinity */ \ + V(xvrspip, XVRSPIP, 0xF00002A4) \ + /* VSX Vector Round to Single-Precision Integer toward -Infinity */ \ + V(xvrspim, XVRSPIM, 0xF00002E4) \ + /* VSX Vector Round to Single-Precision Integer toward Zero */ \ + V(xvrspiz, XVRSPIZ, 0xF0000264) \ + /* VSX Vector Round to Single-Precision Integer */ \ + V(xvrspi, XVRSPI, 0xF0000224) \ + /* VSX Vector Convert Signed Fixed-Point Doubleword to Double-Precision */ \ + V(xvcvsxddp, XVCVSXDDP, 0xF00007E0) \ + /* VSX Vector Convert Unsigned Fixed-Point Doubleword to Double- */ \ + /* Precision */ \ + V(xvcvuxddp, XVCVUXDDP, 0xF00007A0) \ + /* VSX Vector Convert Single-Precision to Double-Precision */ \ + V(xvcvspdp, XVCVSPDP, 0xF0000724) \ + /* VSX Vector Convert Double-Precision to Single-Precision */ \ + V(xvcvdpsp, XVCVDPSP, 0xF0000624) \ + /* VSX Vector Convert Double-Precision to Signed Fixed-Point Word */ \ + /* Saturate */ \ + V(xvcvdpsxws, XVCVDPSXWS, 0xF0000360) \ + /* VSX Vector Convert Double-Precision to Unsigned Fixed-Point Word */ \ + /* Saturate */ \ + V(xvcvdpuxws, XVCVDPUXWS, 0xF0000320) #define PPC_XX2_OPCODE_UNUSED_LIST(V) \ /* VSX Scalar Square Root Double-Precision */ \ @@ -412,14 +427,6 @@ using Instr = uint32_t; V(xsrsqrtesp, XSRSQRTESP, 0xF0000028) \ /* VSX Scalar Square Root Single-Precision */ \ V(xssqrtsp, XSSQRTSP, 0xF000002C) \ - /* Move To VSR Doubleword */ \ - V(mtvsrd, MTVSRD, 0x7C000166) \ - /* Move To VSR Double Doubleword */ \ - V(mtvsrdd, MTVSRDD, 0x7C000366) \ - /* Move To VSR Word Algebraic */ \ - V(mtvsrwa, MTVSRWA, 0x7C0001A6) \ - /* Move To VSR Word and Zero */ \ - V(mtvsrwz, MTVSRWZ, 0x7C0001E6) \ /* VSX Scalar Absolute Value Double-Precision */ \ V(xsabsdp, XSABSDP, 0xF0000564) \ /* VSX Scalar Convert Double-Precision to Single-Precision */ \ @@ -475,37 +482,22 @@ using Instr = uint32_t; V(xsrsqrtedp, XSRSQRTEDP, 0xF0000128) \ /* VSX Scalar Test for software Square Root Double-Precision */ \ V(xstsqrtdp, XSTSQRTDP, 0xF00001A8) \ - /* VSX Vector Convert Double-Precision to Single-Precision */ \ - V(xvcvdpsp, XVCVDPSP, 0xF0000624) \ /* VSX Vector Convert Double-Precision to Signed Fixed-Point Doubleword */ \ /* Saturate */ \ V(xvcvdpsxds, XVCVDPSXDS, 0xF0000760) \ - /* VSX Vector Convert Double-Precision to Signed Fixed-Point Word */ \ - /* Saturate */ \ - V(xvcvdpsxws, XVCVDPSXWS, 0xF0000360) \ /* VSX Vector Convert Double-Precision to Unsigned Fixed-Point */ \ /* Doubleword Saturate */ \ V(xvcvdpuxds, XVCVDPUXDS, 0xF0000720) \ - /* VSX Vector Convert Double-Precision to Unsigned Fixed-Point Word */ \ - /* Saturate */ \ - V(xvcvdpuxws, XVCVDPUXWS, 0xF0000320) \ - /* VSX Vector Convert Single-Precision to Double-Precision */ \ - V(xvcvspdp, XVCVSPDP, 0xF0000724) \ /* VSX Vector Convert Single-Precision to Signed Fixed-Point Doubleword */ \ /* Saturate */ \ V(xvcvspsxds, XVCVSPSXDS, 0xF0000660) \ /* VSX Vector Convert Single-Precision to Unsigned Fixed-Point */ \ /* Doubleword Saturate */ \ V(xvcvspuxds, XVCVSPUXDS, 0xF0000620) \ - /* VSX Vector Convert Signed Fixed-Point Doubleword to Double-Precision */ \ - V(xvcvsxddp, XVCVSXDDP, 0xF00007E0) \ /* VSX Vector Convert Signed Fixed-Point Doubleword to Single-Precision */ \ V(xvcvsxdsp, XVCVSXDSP, 0xF00006E0) \ /* VSX Vector Convert Signed Fixed-Point Word to Double-Precision */ \ V(xvcvsxwdp, XVCVSXWDP, 0xF00003E0) \ - /* VSX Vector Convert Unsigned Fixed-Point Doubleword to Double- */ \ - /* Precision */ \ - V(xvcvuxddp, XVCVUXDDP, 0xF00007A0) \ /* VSX Vector Convert Unsigned Fixed-Point Doubleword to Single- */ \ /* Precision */ \ V(xvcvuxdsp, XVCVUXDSP, 0xF00006A0) \ @@ -528,7 +520,9 @@ using Instr = uint32_t; /* VSX Vector Test for software Square Root Single-Precision */ \ V(xvtsqrtsp, XVTSQRTSP, 0xF00002A8) \ /* Vector Splat Immediate Byte */ \ - V(xxspltib, XXSPLTIB, 0xF00002D0) + V(xxspltib, XXSPLTIB, 0xF00002D0) \ + /* Vector Byte-Reverse Quadword */ \ + V(xxbrq, XXBRQ, 0xF000076C) #define PPC_XX2_OPCODE_LIST(V) \ PPC_XX2_OPCODE_A_FORM_LIST(V) \ @@ -1993,6 +1987,14 @@ using Instr = uint32_t; V(lxvdsx, LXVDSX, 0x7C000298) \ /* Load VSR Vector Word*4 Indexed */ \ V(lxvw, LXVW, 0x7C000618) \ + /* Move To VSR Doubleword */ \ + V(mtvsrd, MTVSRD, 0x7C000166) \ + /* Move To VSR Double Doubleword */ \ + V(mtvsrdd, MTVSRDD, 0x7C000366) \ + /* Move To VSR Word Algebraic */ \ + V(mtvsrwa, MTVSRWA, 0x7C0001A6) \ + /* Move To VSR Word and Zero */ \ + V(mtvsrwz, MTVSRWZ, 0x7C0001E6) \ /* Move From VSR Doubleword */ \ V(mfvsrd, MFVSRD, 0x7C000066) \ /* Move From VSR Word and Zero */ \ @@ -2289,6 +2291,14 @@ using Instr = uint32_t; V(vmulouh, VMULOUH, 0x10000048) \ /* Vector Multiply Odd Signed Halfword */ \ V(vmulosh, VMULOSH, 0x10000148) \ + /* Vector Multiply Even Signed Word */ \ + V(vmulesw, VMULESW, 0x10000388) \ + /* Vector Multiply Even Unsigned Word */ \ + V(vmuleuw, VMULEUW, 0x10000288) \ + /* Vector Multiply Odd Signed Word */ \ + V(vmulosw, VMULOSW, 0x10000188) \ + /* Vector Multiply Odd Unsigned Word */ \ + V(vmulouw, VMULOUW, 0x10000088) \ /* Vector Sum across Quarter Signed Halfword Saturate */ \ V(vsum4shs, VSUM4SHS, 0x10000648) \ /* Vector Pack Unsigned Word Unsigned Saturate */ \ @@ -2390,7 +2400,19 @@ using Instr = uint32_t; /* Vector Maximum Single-Precision */ \ V(vmaxfp, VMAXFP, 0x1000040A) \ /* Vector Bit Permute Quadword */ \ - V(vbpermq, VBPERMQ, 0x1000054C) + V(vbpermq, VBPERMQ, 0x1000054C) \ + /* Vector Merge High Byte */ \ + V(vmrghb, VMRGHB, 0x1000000C) \ + /* Vector Merge High Halfword */ \ + V(vmrghh, VMRGHH, 0x1000004C) \ + /* Vector Merge High Word */ \ + V(vmrghw, VMRGHW, 0x1000008C) \ + /* Vector Merge Low Byte */ \ + V(vmrglb, VMRGLB, 0x1000010C) \ + /* Vector Merge Low Halfword */ \ + V(vmrglh, VMRGLH, 0x1000014C) \ + /* Vector Merge Low Word */ \ + V(vmrglw, VMRGLW, 0x1000018C) #define PPC_VX_OPCODE_C_FORM_LIST(V) \ /* Vector Unpack Low Signed Word */ \ @@ -2404,7 +2426,9 @@ using Instr = uint32_t; /* Vector Unpack Low Signed Byte */ \ V(vupklsb, VUPKLSB, 0x1000028E) \ /* Vector Unpack High Signed Byte */ \ - V(vupkhsb, VUPKHSB, 0x1000020E) + V(vupkhsb, VUPKHSB, 0x1000020E) \ + /* Vector Population Count Byte */ \ + V(vpopcntb, VPOPCNTB, 0x10000703) #define PPC_VX_OPCODE_UNUSED_LIST(V) \ /* Decimal Add Modulo */ \ @@ -2459,26 +2483,6 @@ using Instr = uint32_t; V(vgbbd, VGBBD, 0x1000050C) \ /* Vector Log Base 2 Estimate Single-Precision */ \ V(vlogefp, VLOGEFP, 0x100001CA) \ - /* Vector Merge High Byte */ \ - V(vmrghb, VMRGHB, 0x1000000C) \ - /* Vector Merge High Halfword */ \ - V(vmrghh, VMRGHH, 0x1000004C) \ - /* Vector Merge High Word */ \ - V(vmrghw, VMRGHW, 0x1000008C) \ - /* Vector Merge Low Byte */ \ - V(vmrglb, VMRGLB, 0x1000010C) \ - /* Vector Merge Low Halfword */ \ - V(vmrglh, VMRGLH, 0x1000014C) \ - /* Vector Merge Low Word */ \ - V(vmrglw, VMRGLW, 0x1000018C) \ - /* Vector Multiply Even Signed Word */ \ - V(vmulesw, VMULESW, 0x10000388) \ - /* Vector Multiply Even Unsigned Word */ \ - V(vmuleuw, VMULEUW, 0x10000288) \ - /* Vector Multiply Odd Signed Word */ \ - V(vmulosw, VMULOSW, 0x10000188) \ - /* Vector Multiply Odd Unsigned Word */ \ - V(vmulouw, VMULOUW, 0x10000088) \ /* Vector NAND */ \ V(vnand, VNAND, 0x10000584) \ /* Vector OR with Complement */ \ @@ -2503,8 +2507,6 @@ using Instr = uint32_t; V(vpmsumh, VPMSUMH, 0x10000448) \ /* Vector Polynomial Multiply-Sum Word */ \ V(vpmsumw, VPMSUMW, 0x10000488) \ - /* Vector Population Count Byte */ \ - V(vpopcntb, VPOPCNTB, 0x10000703) \ /* Vector Population Count Doubleword */ \ V(vpopcntd, VPOPCNTD, 0x100007C3) \ /* Vector Population Count Halfword */ \ @@ -2912,7 +2914,11 @@ class Instruction { PPC_M_OPCODE_LIST(OPCODE_CASES) return static_cast<Opcode>(opcode); } - + opcode = extcode | BitField(5, 0); + switch (opcode) { + PPC_VA_OPCODE_LIST(OPCODE_CASES) + return static_cast<Opcode>(opcode); + } opcode = extcode | BitField(10, 0); switch (opcode) { PPC_VX_OPCODE_LIST(OPCODE_CASES) @@ -2929,13 +2935,17 @@ class Instruction { PPC_XFX_OPCODE_LIST(OPCODE_CASES) return static_cast<Opcode>(opcode); } + opcode = extcode | BitField(10, 2); + switch (opcode) { + PPC_XX2_OPCODE_LIST(OPCODE_CASES) + return static_cast<Opcode>(opcode); + } opcode = extcode | BitField(10, 1); switch (opcode) { PPC_X_OPCODE_LIST(OPCODE_CASES) PPC_XL_OPCODE_LIST(OPCODE_CASES) PPC_XFL_OPCODE_LIST(OPCODE_CASES) PPC_XX1_OPCODE_LIST(OPCODE_CASES) - PPC_XX2_OPCODE_LIST(OPCODE_CASES) PPC_EVX_OPCODE_LIST(OPCODE_CASES) return static_cast<Opcode>(opcode); } @@ -2961,11 +2971,6 @@ class Instruction { PPC_Z23_OPCODE_LIST(OPCODE_CASES) return static_cast<Opcode>(opcode); } - opcode = extcode | BitField(5, 0); - switch (opcode) { - PPC_VA_OPCODE_LIST(OPCODE_CASES) - return static_cast<Opcode>(opcode); - } opcode = extcode | BitField(5, 1); switch (opcode) { PPC_A_OPCODE_LIST(OPCODE_CASES) diff --git a/deps/v8/src/codegen/ppc/macro-assembler-ppc.cc b/deps/v8/src/codegen/ppc/macro-assembler-ppc.cc index e78130ee42..658a41f381 100644 --- a/deps/v8/src/codegen/ppc/macro-assembler-ppc.cc +++ b/deps/v8/src/codegen/ppc/macro-assembler-ppc.cc @@ -22,7 +22,10 @@ #include "src/runtime/runtime.h" #include "src/snapshot/embedded/embedded-data.h" #include "src/snapshot/snapshot.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY // Satisfy cpplint check, but don't include platform-specific header. It is // included recursively via macro-assembler.h. @@ -1792,8 +1795,13 @@ void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone, // Put input on stack. stfdu(double_input, MemOperand(sp, -kDoubleSize)); +#if V8_ENABLE_WEBASSEMBLY if (stub_mode == StubCallMode::kCallWasmRuntimeStub) { Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL); +#else + // For balance. + if (false) { +#endif // V8_ENABLE_WEBASSEMBLY } else { Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET); } @@ -1967,7 +1975,7 @@ void TurboAssembler::LoadMap(Register destination, Register object) { FieldMemOperand(object, HeapObject::kMapOffset)); } -void MacroAssembler::LoadNativeContextSlot(int index, Register dst) { +void MacroAssembler::LoadNativeContextSlot(Register dst, int index) { LoadMap(dst, cp); LoadTaggedPointerField( dst, FieldMemOperand( diff --git a/deps/v8/src/codegen/ppc/macro-assembler-ppc.h b/deps/v8/src/codegen/ppc/macro-assembler-ppc.h index 5da219ba84..1d8f3a388d 100644 --- a/deps/v8/src/codegen/ppc/macro-assembler-ppc.h +++ b/deps/v8/src/codegen/ppc/macro-assembler-ppc.h @@ -119,6 +119,12 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { // Returns the pc offset at which the frame ends. int LeaveFrame(StackFrame::Type type, int stack_adjustment = 0); + void AllocateStackSpace(int bytes) { + DCHECK_GE(bytes, 0); + if (bytes == 0) return; + Add(sp, sp, -bytes, r0); + } + // Push a fixed frame, consisting of lr, fp, constant pool. void PushCommonFrame(Register marker_reg = no_reg); @@ -781,10 +787,10 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // Load the global proxy from the current context. void LoadGlobalProxy(Register dst) { - LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst); + LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX); } - void LoadNativeContextSlot(int index, Register dst); + void LoadNativeContextSlot(Register dst, int index); // ---------------------------------------------------------------- // new PPC macro-assembler interfaces that are slightly higher level diff --git a/deps/v8/src/codegen/ppc/register-ppc.h b/deps/v8/src/codegen/ppc/register-ppc.h index bbbee603de..ffeb327055 100644 --- a/deps/v8/src/codegen/ppc/register-ppc.h +++ b/deps/v8/src/codegen/ppc/register-ppc.h @@ -213,7 +213,12 @@ constexpr Register kConstantPoolRegister = r28; // Constant pool. constexpr Register kRootRegister = r29; // Roots array pointer. constexpr Register cp = r30; // JavaScript context pointer. -constexpr bool kPadArguments = false; +// Returns the number of padding slots needed for stack pointer alignment. +constexpr int ArgumentPaddingSlots(int argument_count) { + // No argument padding required. + return 0; +} + constexpr bool kSimpleFPAliasing = true; constexpr bool kSimdMaskRegisters = false; @@ -249,6 +254,29 @@ static_assert(sizeof(DoubleRegister) == sizeof(int), using FloatRegister = DoubleRegister; +// | | 0 +// | | 1 +// | | 2 +// | | ... +// | | 31 +// VSX | +// | | 32 +// | | 33 +// | VMX | 34 +// | | ... +// | | 63 +// +// VSX registers (0 to 63) can be used by VSX vector instructions, which are +// mainly focused on Floating Point arithmetic. They do have few Integer +// Instructions such as logical operations, merge and select. The main Simd +// integer instructions such as add/sub/mul/ extract_lane/replace_lane, +// comparisons etc. are only available with VMX instructions and can only access +// the VMX set of vector registers (which is a subset of VSX registers). So to +// assure access to all Simd instructions in V8 and avoid moving data between +// registers, we are only using the upper 32 registers (VMX set) for Simd +// operations and only use the lower set for scalar (non simd) floating point +// operations which makes our Simd register set separate from Floating Point +// ones. enum Simd128RegisterCode { #define REGISTER_CODE(R) kSimd128Code_##R, SIMD128_REGISTERS(REGISTER_CODE) diff --git a/deps/v8/src/codegen/register-arch.h b/deps/v8/src/codegen/register-arch.h index 3a72daae27..3936ee80cc 100644 --- a/deps/v8/src/codegen/register-arch.h +++ b/deps/v8/src/codegen/register-arch.h @@ -30,4 +30,18 @@ #error Unknown architecture. #endif +namespace v8 { +namespace internal { + +constexpr int AddArgumentPaddingSlots(int argument_count) { + return argument_count + ArgumentPaddingSlots(argument_count); +} + +constexpr bool ShouldPadArguments(int argument_count) { + return ArgumentPaddingSlots(argument_count) != 0; +} + +} // namespace internal +} // namespace v8 + #endif // V8_CODEGEN_REGISTER_ARCH_H_ diff --git a/deps/v8/src/codegen/register.cc b/deps/v8/src/codegen/register.cc deleted file mode 100644 index 4ad76c6caa..0000000000 --- a/deps/v8/src/codegen/register.cc +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2019 the V8 project authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "src/codegen/register.h" -#include "src/codegen/register-arch.h" - -namespace v8 { -namespace internal { - -bool ShouldPadArguments(int argument_count) { - return kPadArguments && (argument_count % 2 != 0); -} - -} // namespace internal -} // namespace v8 diff --git a/deps/v8/src/codegen/register.h b/deps/v8/src/codegen/register.h index 2dcf0fbe8f..57f3a1c62a 100644 --- a/deps/v8/src/codegen/register.h +++ b/deps/v8/src/codegen/register.h @@ -70,9 +70,6 @@ class RegisterBase { int reg_code_; }; -// Whether padding is needed for the given stack argument count. -bool ShouldPadArguments(int argument_count); - template <typename RegType, typename = decltype(RegisterName(std::declval<RegType>()))> inline std::ostream& operator<<(std::ostream& os, RegType reg) { diff --git a/deps/v8/src/codegen/reloc-info.cc b/deps/v8/src/codegen/reloc-info.cc index 25b3daef8e..753b34bdbf 100644 --- a/deps/v8/src/codegen/reloc-info.cc +++ b/deps/v8/src/codegen/reloc-info.cc @@ -519,14 +519,15 @@ void RelocInfo::Verify(Isolate* isolate) { Address target = target_internal_reference(); Address pc = target_internal_reference_address(); Code code = Code::cast(isolate->FindCodeObject(pc)); - CHECK(target >= code.InstructionStart()); - CHECK(target <= code.InstructionEnd()); + CHECK(target >= code.InstructionStart(isolate, pc)); + CHECK(target <= code.InstructionEnd(isolate, pc)); break; } case OFF_HEAP_TARGET: { Address addr = target_off_heap_target(); CHECK_NE(addr, kNullAddress); - CHECK(!InstructionStream::TryLookupCode(isolate, addr).is_null()); + CHECK(Builtins::IsBuiltinId( + InstructionStream::TryLookupCode(isolate, addr))); break; } case RUNTIME_ENTRY: diff --git a/deps/v8/src/codegen/reloc-info.h b/deps/v8/src/codegen/reloc-info.h index f478de86a1..bef433e10b 100644 --- a/deps/v8/src/codegen/reloc-info.h +++ b/deps/v8/src/codegen/reloc-info.h @@ -65,6 +65,9 @@ class RelocInfo { WASM_CALL, // FIRST_SHAREABLE_RELOC_MODE WASM_STUB_CALL, + // TODO(ishell): rename to UNEMBEDDED_BUILTIN_ENTRY. + // An un-embedded off-heap instruction stream target. + // See http://crbug.com/v8/11527 for details. RUNTIME_ENTRY, EXTERNAL_REFERENCE, // The address of an external C++ function. @@ -148,6 +151,7 @@ class RelocInfo { return base::IsInRange(mode, FIRST_EMBEDDED_OBJECT_RELOC_MODE, LAST_EMBEDDED_OBJECT_RELOC_MODE); } + // TODO(ishell): rename to IsUnembeddedBuiltinEntry(). static constexpr bool IsRuntimeEntry(Mode mode) { return mode == RUNTIME_ENTRY; } diff --git a/deps/v8/src/codegen/riscv64/assembler-riscv64-inl.h b/deps/v8/src/codegen/riscv64/assembler-riscv64-inl.h index b99262cb36..40bd56d15b 100644 --- a/deps/v8/src/codegen/riscv64/assembler-riscv64-inl.h +++ b/deps/v8/src/codegen/riscv64/assembler-riscv64-inl.h @@ -45,8 +45,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return IsSupported(FPU); } -bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(RISCV_SIMD); } - // ----------------------------------------------------------------------------- // Operand and MemOperand. diff --git a/deps/v8/src/codegen/riscv64/assembler-riscv64.cc b/deps/v8/src/codegen/riscv64/assembler-riscv64.cc index e070e72f45..914ea26f9f 100644 --- a/deps/v8/src/codegen/riscv64/assembler-riscv64.cc +++ b/deps/v8/src/codegen/riscv64/assembler-riscv64.cc @@ -60,6 +60,8 @@ static unsigned CpuFeaturesImpliedByCompiler() { return answer; } +bool CpuFeatures::SupportsWasmSimd128() { return IsSupported(RISCV_SIMD); } + void CpuFeatures::ProbeImpl(bool cross_compile) { supported_ |= CpuFeaturesImpliedByCompiler(); diff --git a/deps/v8/src/codegen/riscv64/interface-descriptors-riscv64.cc b/deps/v8/src/codegen/riscv64/interface-descriptors-riscv64.cc index 26730aceca..23953097cd 100644 --- a/deps/v8/src/codegen/riscv64/interface-descriptors-riscv64.cc +++ b/deps/v8/src/codegen/riscv64/interface-descriptors-riscv64.cc @@ -281,6 +281,18 @@ void ResumeGeneratorDescriptor::InitializePlatformSpecific( data->InitializePlatformSpecific(arraysize(registers), registers); } +void BinaryOp_BaselineDescriptor::InitializePlatformSpecific( + CallInterfaceDescriptorData* data) { + // TODO(v8:11421): Implement on this platform. + InitializePlatformUnimplemented(data, kParameterCount); +} + +void Compare_BaselineDescriptor::InitializePlatformSpecific( + CallInterfaceDescriptorData* data) { + // TODO(v8:11421): Implement on this platform. + InitializePlatformUnimplemented(data, kParameterCount); +} + void FrameDropperTrampolineDescriptor::InitializePlatformSpecific( CallInterfaceDescriptorData* data) { Register registers[] = { diff --git a/deps/v8/src/codegen/riscv64/macro-assembler-riscv64.cc b/deps/v8/src/codegen/riscv64/macro-assembler-riscv64.cc index a479666120..ff798da0e9 100644 --- a/deps/v8/src/codegen/riscv64/macro-assembler-riscv64.cc +++ b/deps/v8/src/codegen/riscv64/macro-assembler-riscv64.cc @@ -3862,7 +3862,7 @@ void TurboAssembler::LoadMap(Register destination, Register object) { Ld(destination, FieldMemOperand(object, HeapObject::kMapOffset)); } -void MacroAssembler::LoadNativeContextSlot(int index, Register dst) { +void MacroAssembler::LoadNativeContextSlot(Register dst, int index) { LoadMap(dst, cp); Ld(dst, FieldMemOperand(dst, Map::kConstructorOrBackPointerOrNativeContextOffset)); @@ -4564,7 +4564,8 @@ void TurboAssembler::CallCodeObject(Register code_object) { Call(code_object); } -void TurboAssembler::JumpCodeObject(Register code_object) { +void TurboAssembler::JumpCodeObject(Register code_object, JumpMode jump_mode) { + DCHECK_EQ(JumpMode::kJump, jump_mode); LoadCodeObjectEntry(code_object, code_object); Jump(code_object); } diff --git a/deps/v8/src/codegen/riscv64/macro-assembler-riscv64.h b/deps/v8/src/codegen/riscv64/macro-assembler-riscv64.h index 75c03cc27b..b260f1c200 100644 --- a/deps/v8/src/codegen/riscv64/macro-assembler-riscv64.h +++ b/deps/v8/src/codegen/riscv64/macro-assembler-riscv64.h @@ -150,6 +150,14 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { #undef COND_TYPED_ARGS #undef COND_ARGS + void AllocateStackSpace(Register bytes) { Sub64(sp, sp, bytes); } + + void AllocateStackSpace(int bytes) { + DCHECK_GE(bytes, 0); + if (bytes == 0) return; + Sub64(sp, sp, Operand(bytes)); + } + inline void NegateBool(Register rd, Register rs) { Xor(rd, rs, 1); } // Compare float, if any operand is NaN, result is false except for NE @@ -219,7 +227,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void LoadCodeObjectEntry(Register destination, Register code_object) override; void CallCodeObject(Register code_object) override; - void JumpCodeObject(Register code_object) override; + void JumpCodeObject(Register code_object, + JumpMode jump_mode = JumpMode::kJump) override; // Generates an instruction sequence s.t. the return address points to the // instruction following the call. @@ -986,10 +995,10 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // Load the global proxy from the current context. void LoadGlobalProxy(Register dst) { - LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst); + LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX); } - void LoadNativeContextSlot(int index, Register dst); + void LoadNativeContextSlot(Register dst, int index); // Load the initial map from the global function. The registers // function and map can be the same, function is then overwritten. diff --git a/deps/v8/src/codegen/riscv64/register-riscv64.h b/deps/v8/src/codegen/riscv64/register-riscv64.h index 2626c4eae7..b97594becd 100644 --- a/deps/v8/src/codegen/riscv64/register-riscv64.h +++ b/deps/v8/src/codegen/riscv64/register-riscv64.h @@ -41,6 +41,12 @@ namespace internal { V(ft4) V(ft5) V(ft6) V(ft7) V(fa0) V(fa1) V(fa2) V(fa3) V(fa4) V(fa5) \ V(fa6) V(fa7) +// Returns the number of padding slots needed for stack pointer alignment. +constexpr int ArgumentPaddingSlots(int argument_count) { + // No argument padding required. + return 0; +} + // clang-format on // Note that the bit values must match those used in actual instruction diff --git a/deps/v8/src/codegen/s390/assembler-s390-inl.h b/deps/v8/src/codegen/s390/assembler-s390-inl.h index f58e891a90..dc04acec61 100644 --- a/deps/v8/src/codegen/s390/assembler-s390-inl.h +++ b/deps/v8/src/codegen/s390/assembler-s390-inl.h @@ -48,10 +48,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return true; } -bool CpuFeatures::SupportsWasmSimd128() { - return CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_1); -} - void RelocInfo::apply(intptr_t delta) { // Absolute code pointer inside code object moves with the code object. if (IsInternalReference(rmode_)) { diff --git a/deps/v8/src/codegen/s390/assembler-s390.cc b/deps/v8/src/codegen/s390/assembler-s390.cc index 76b3d9953e..dd5f59bc0b 100644 --- a/deps/v8/src/codegen/s390/assembler-s390.cc +++ b/deps/v8/src/codegen/s390/assembler-s390.cc @@ -159,6 +159,10 @@ static bool supportsSTFLE() { #endif } +bool CpuFeatures::SupportsWasmSimd128() { + return CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_1); +} + void CpuFeatures::ProbeImpl(bool cross_compile) { supported_ |= CpuFeaturesImpliedByCompiler(); icache_line_size_ = 256; diff --git a/deps/v8/src/codegen/s390/macro-assembler-s390.cc b/deps/v8/src/codegen/s390/macro-assembler-s390.cc index 511649af80..be5798d8d4 100644 --- a/deps/v8/src/codegen/s390/macro-assembler-s390.cc +++ b/deps/v8/src/codegen/s390/macro-assembler-s390.cc @@ -23,7 +23,10 @@ #include "src/runtime/runtime.h" #include "src/snapshot/embedded/embedded-data.h" #include "src/snapshot/snapshot.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY // Satisfy cpplint check, but don't include platform-specific header. It is // included recursively via macro-assembler.h. @@ -324,8 +327,6 @@ void TurboAssembler::Jump(intptr_t target, RelocInfo::Mode rmode, if (cond != al) b(NegateCondition(cond), &skip); - DCHECK(rmode == RelocInfo::CODE_TARGET || rmode == RelocInfo::RUNTIME_ENTRY); - mov(ip, Operand(target, rmode)); b(ip); @@ -1838,8 +1839,13 @@ void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone, lay(sp, MemOperand(sp, -kDoubleSize)); StoreF64(double_input, MemOperand(sp)); +#if V8_ENABLE_WEBASSEMBLY if (stub_mode == StubCallMode::kCallWasmRuntimeStub) { Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL); +#else + // For balance. + if (false) { +#endif // V8_ENABLE_WEBASSEMBLY } else { Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET); } @@ -2002,7 +2008,7 @@ void TurboAssembler::LoadMap(Register destination, Register object) { FieldMemOperand(object, HeapObject::kMapOffset)); } -void MacroAssembler::LoadNativeContextSlot(int index, Register dst) { +void MacroAssembler::LoadNativeContextSlot(Register dst, int index) { LoadMap(dst, cp); LoadTaggedPointerField( dst, FieldMemOperand( @@ -4755,6 +4761,218 @@ void TurboAssembler::CountTrailingZerosU64(Register dst, Register src, bind(&done); } +void TurboAssembler::AtomicCmpExchangeHelper(Register addr, Register output, + Register old_value, + Register new_value, int start, + int end, int shift_amount, + int offset, Register temp0, + Register temp1) { + LoadU32(temp0, MemOperand(addr, offset)); + llgfr(temp1, temp0); + RotateInsertSelectBits(temp0, old_value, Operand(start), Operand(end), + Operand(shift_amount), false); + RotateInsertSelectBits(temp1, new_value, Operand(start), Operand(end), + Operand(shift_amount), false); + CmpAndSwap(temp0, temp1, MemOperand(addr, offset)); + RotateInsertSelectBits(output, temp0, Operand(start + shift_amount), + Operand(end + shift_amount), + Operand(64 - shift_amount), true); +} + +void TurboAssembler::AtomicCmpExchangeU8(Register addr, Register output, + Register old_value, Register new_value, + Register temp0, Register temp1) { +#ifdef V8_TARGET_BIG_ENDIAN +#define ATOMIC_COMP_EXCHANGE_BYTE(i) \ + { \ + constexpr int idx = (i); \ + static_assert(idx <= 3 && idx >= 0, "idx is out of range!"); \ + constexpr int start = 32 + 8 * idx; \ + constexpr int end = start + 7; \ + constexpr int shift_amount = (3 - idx) * 8; \ + AtomicCmpExchangeHelper(addr, output, old_value, new_value, start, end, \ + shift_amount, -idx, temp0, temp1); \ + } +#else +#define ATOMIC_COMP_EXCHANGE_BYTE(i) \ + { \ + constexpr int idx = (i); \ + static_assert(idx <= 3 && idx >= 0, "idx is out of range!"); \ + constexpr int start = 32 + 8 * (3 - idx); \ + constexpr int end = start + 7; \ + constexpr int shift_amount = idx * 8; \ + AtomicCmpExchangeHelper(addr, output, old_value, new_value, start, end, \ + shift_amount, -idx, temp0, temp1); \ + } +#endif + + Label one, two, three, done; + tmll(addr, Operand(3)); + b(Condition(1), &three); + b(Condition(2), &two); + b(Condition(4), &one); + /* ending with 0b00 */ + ATOMIC_COMP_EXCHANGE_BYTE(0); + b(&done); + /* ending with 0b01 */ + bind(&one); + ATOMIC_COMP_EXCHANGE_BYTE(1); + b(&done); + /* ending with 0b10 */ + bind(&two); + ATOMIC_COMP_EXCHANGE_BYTE(2); + b(&done); + /* ending with 0b11 */ + bind(&three); + ATOMIC_COMP_EXCHANGE_BYTE(3); + bind(&done); +} + +void TurboAssembler::AtomicCmpExchangeU16(Register addr, Register output, + Register old_value, + Register new_value, Register temp0, + Register temp1) { +#ifdef V8_TARGET_BIG_ENDIAN +#define ATOMIC_COMP_EXCHANGE_HALFWORD(i) \ + { \ + constexpr int idx = (i); \ + static_assert(idx <= 1 && idx >= 0, "idx is out of range!"); \ + constexpr int start = 32 + 16 * idx; \ + constexpr int end = start + 15; \ + constexpr int shift_amount = (1 - idx) * 16; \ + AtomicCmpExchangeHelper(addr, output, old_value, new_value, start, end, \ + shift_amount, -idx * 2, temp0, temp1); \ + } +#else +#define ATOMIC_COMP_EXCHANGE_HALFWORD(i) \ + { \ + constexpr int idx = (i); \ + static_assert(idx <= 1 && idx >= 0, "idx is out of range!"); \ + constexpr int start = 32 + 16 * (1 - idx); \ + constexpr int end = start + 15; \ + constexpr int shift_amount = idx * 16; \ + AtomicCmpExchangeHelper(addr, output, old_value, new_value, start, end, \ + shift_amount, -idx * 2, temp0, temp1); \ + } +#endif + + Label two, done; + tmll(addr, Operand(3)); + b(Condition(2), &two); + ATOMIC_COMP_EXCHANGE_HALFWORD(0); + b(&done); + bind(&two); + ATOMIC_COMP_EXCHANGE_HALFWORD(1); + bind(&done); +} + +void TurboAssembler::AtomicExchangeHelper(Register addr, Register value, + Register output, int start, int end, + int shift_amount, int offset, + Register scratch) { + Label do_cs; + LoadU32(output, MemOperand(addr, offset)); + bind(&do_cs); + llgfr(scratch, output); + RotateInsertSelectBits(scratch, value, Operand(start), Operand(end), + Operand(shift_amount), false); + csy(output, scratch, MemOperand(addr, offset)); + bne(&do_cs, Label::kNear); + srl(output, Operand(shift_amount)); +} + +void TurboAssembler::AtomicExchangeU8(Register addr, Register value, + Register output, Register scratch) { +#ifdef V8_TARGET_BIG_ENDIAN +#define ATOMIC_EXCHANGE_BYTE(i) \ + { \ + constexpr int idx = (i); \ + static_assert(idx <= 3 && idx >= 0, "idx is out of range!"); \ + constexpr int start = 32 + 8 * idx; \ + constexpr int end = start + 7; \ + constexpr int shift_amount = (3 - idx) * 8; \ + AtomicExchangeHelper(addr, value, output, start, end, shift_amount, -idx, \ + scratch); \ + } +#else +#define ATOMIC_EXCHANGE_BYTE(i) \ + { \ + constexpr int idx = (i); \ + static_assert(idx <= 3 && idx >= 0, "idx is out of range!"); \ + constexpr int start = 32 + 8 * (3 - idx); \ + constexpr int end = start + 7; \ + constexpr int shift_amount = idx * 8; \ + AtomicExchangeHelper(addr, value, output, start, end, shift_amount, -idx, \ + scratch); \ + } +#endif + Label three, two, one, done; + tmll(addr, Operand(3)); + b(Condition(1), &three); + b(Condition(2), &two); + b(Condition(4), &one); + + // end with 0b00 + ATOMIC_EXCHANGE_BYTE(0); + b(&done); + + // ending with 0b01 + bind(&one); + ATOMIC_EXCHANGE_BYTE(1); + b(&done); + + // ending with 0b10 + bind(&two); + ATOMIC_EXCHANGE_BYTE(2); + b(&done); + + // ending with 0b11 + bind(&three); + ATOMIC_EXCHANGE_BYTE(3); + + bind(&done); +} + +void TurboAssembler::AtomicExchangeU16(Register addr, Register value, + Register output, Register scratch) { +#ifdef V8_TARGET_BIG_ENDIAN +#define ATOMIC_EXCHANGE_HALFWORD(i) \ + { \ + constexpr int idx = (i); \ + static_assert(idx <= 1 && idx >= 0, "idx is out of range!"); \ + constexpr int start = 32 + 16 * idx; \ + constexpr int end = start + 15; \ + constexpr int shift_amount = (1 - idx) * 16; \ + AtomicExchangeHelper(addr, value, output, start, end, shift_amount, \ + -idx * 2, scratch); \ + } +#else +#define ATOMIC_EXCHANGE_HALFWORD(i) \ + { \ + constexpr int idx = (i); \ + static_assert(idx <= 1 && idx >= 0, "idx is out of range!"); \ + constexpr int start = 32 + 16 * (1 - idx); \ + constexpr int end = start + 15; \ + constexpr int shift_amount = idx * 16; \ + AtomicExchangeHelper(addr, value, output, start, end, shift_amount, \ + -idx * 2, scratch); \ + } +#endif + Label two, done; + tmll(addr, Operand(3)); + b(Condition(2), &two); + + // end with 0b00 + ATOMIC_EXCHANGE_HALFWORD(0); + b(&done); + + // ending with 0b10 + bind(&two); + ATOMIC_EXCHANGE_HALFWORD(1); + + bind(&done); +} + } // namespace internal } // namespace v8 diff --git a/deps/v8/src/codegen/s390/macro-assembler-s390.h b/deps/v8/src/codegen/s390/macro-assembler-s390.h index f4c3d038b3..f2719c3086 100644 --- a/deps/v8/src/codegen/s390/macro-assembler-s390.h +++ b/deps/v8/src/codegen/s390/macro-assembler-s390.h @@ -46,6 +46,22 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { public: using TurboAssemblerBase::TurboAssemblerBase; + void AtomicCmpExchangeHelper(Register addr, Register output, + Register old_value, Register new_value, + int start, int end, int shift_amount, int offset, + Register temp0, Register temp1); + void AtomicCmpExchangeU8(Register addr, Register output, Register old_value, + Register new_value, Register temp0, Register temp1); + void AtomicCmpExchangeU16(Register addr, Register output, Register old_value, + Register new_value, Register temp0, Register temp1); + void AtomicExchangeHelper(Register addr, Register value, Register output, + int start, int end, int shift_amount, int offset, + Register scratch); + void AtomicExchangeU8(Register addr, Register value, Register output, + Register scratch); + void AtomicExchangeU16(Register addr, Register value, Register output, + Register scratch); + void DoubleMax(DoubleRegister result_reg, DoubleRegister left_reg, DoubleRegister right_reg); void DoubleMin(DoubleRegister result_reg, DoubleRegister left_reg, @@ -977,6 +993,12 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { // Returns the pc offset at which the frame ends. int LeaveFrame(StackFrame::Type type, int stack_adjustment = 0); + void AllocateStackSpace(int bytes) { + DCHECK_GE(bytes, 0); + if (bytes == 0) return; + lay(sp, MemOperand(sp, -bytes)); + } + void CheckPageFlag(Register object, Register scratch, int mask, Condition cc, Label* condition_met); @@ -1239,10 +1261,10 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // Load the global proxy from the current context. void LoadGlobalProxy(Register dst) { - LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst); + LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX); } - void LoadNativeContextSlot(int index, Register dst); + void LoadNativeContextSlot(Register dst, int index); // --------------------------------------------------------------------------- // Smi utilities diff --git a/deps/v8/src/codegen/s390/register-s390.h b/deps/v8/src/codegen/s390/register-s390.h index 0c6da03901..48accf08c5 100644 --- a/deps/v8/src/codegen/s390/register-s390.h +++ b/deps/v8/src/codegen/s390/register-s390.h @@ -167,7 +167,12 @@ constexpr Register no_reg = Register::no_reg(); constexpr Register kRootRegister = r10; // Roots array pointer. constexpr Register cp = r13; // JavaScript context pointer. -constexpr bool kPadArguments = false; +// Returns the number of padding slots needed for stack pointer alignment. +constexpr int ArgumentPaddingSlots(int argument_count) { + // No argument padding required. + return 0; +} + constexpr bool kSimpleFPAliasing = true; constexpr bool kSimdMaskRegisters = false; diff --git a/deps/v8/src/codegen/safepoint-table.cc b/deps/v8/src/codegen/safepoint-table.cc index dd379e0535..58fb6ed9e1 100644 --- a/deps/v8/src/codegen/safepoint-table.cc +++ b/deps/v8/src/codegen/safepoint-table.cc @@ -10,19 +10,24 @@ #include "src/diagnostics/disasm.h" #include "src/execution/frames-inl.h" #include "src/utils/ostreams.h" + +#if V8_ENABLE_WEBASSEMBLY #include "src/wasm/wasm-code-manager.h" +#endif // V8_ENABLE_WEBASSEMBLY namespace v8 { namespace internal { -SafepointTable::SafepointTable(Code code) - : SafepointTable(code.InstructionStart(), code.SafepointTableAddress(), - code.stack_slots(), true) {} +SafepointTable::SafepointTable(Isolate* isolate, Address pc, Code code) + : SafepointTable(code.InstructionStart(isolate, pc), + code.SafepointTableAddress(), code.stack_slots(), true) {} +#if V8_ENABLE_WEBASSEMBLY SafepointTable::SafepointTable(const wasm::WasmCode* code) : SafepointTable(code->instruction_start(), code->instruction_start() + code->safepoint_table_offset(), code->stack_slots(), false) {} +#endif // V8_ENABLE_WEBASSEMBLY SafepointTable::SafepointTable(Address instruction_start, Address safepoint_table_address, @@ -92,7 +97,7 @@ Safepoint SafepointTableBuilder::DefineSafepoint(Assembler* assembler) { deoptimization_info_.push_back( DeoptimizationInfo(zone_, assembler->pc_offset_for_safepoint())); DeoptimizationInfo& new_info = deoptimization_info_.back(); - return Safepoint(new_info.indexes); + return Safepoint(new_info.stack_indexes, &new_info.register_indexes); } unsigned SafepointTableBuilder::GetCodeOffset() const { @@ -143,14 +148,22 @@ void SafepointTableBuilder::Emit(Assembler* assembler, int bits_per_entry) { STATIC_ASSERT(SafepointTable::kFixedEntrySize == 3 * kIntSize); for (const DeoptimizationInfo& info : deoptimization_info_) { assembler->dd(info.pc); - assembler->dd(info.deopt_index); + if (info.register_indexes) { + // We emit the register indexes in the same bits as the deopt_index. + // Register indexes and deopt_index should not exist at the same time. + DCHECK_EQ(info.deopt_index, + static_cast<uint32_t>(Safepoint::kNoDeoptimizationIndex)); + assembler->dd(info.register_indexes); + } else { + assembler->dd(info.deopt_index); + } assembler->dd(info.trampoline); } // Emit table of bitmaps. ZoneVector<uint8_t> bits(bytes_per_entry, 0, zone_); for (const DeoptimizationInfo& info : deoptimization_info_) { - ZoneChunkList<int>* indexes = info.indexes; + ZoneChunkList<int>* indexes = info.stack_indexes; std::fill(bits.begin(), bits.end(), 0); // Run through the indexes and build a bitmap. @@ -194,13 +207,15 @@ bool SafepointTableBuilder::IsIdenticalExceptForPc( const DeoptimizationInfo& info1, const DeoptimizationInfo& info2) const { if (info1.deopt_index != info2.deopt_index) return false; - ZoneChunkList<int>* indexes1 = info1.indexes; - ZoneChunkList<int>* indexes2 = info2.indexes; + ZoneChunkList<int>* indexes1 = info1.stack_indexes; + ZoneChunkList<int>* indexes2 = info2.stack_indexes; if (indexes1->size() != indexes2->size()) return false; if (!std::equal(indexes1->begin(), indexes1->end(), indexes2->begin())) { return false; } + if (info1.register_indexes != info2.register_indexes) return false; + return true; } diff --git a/deps/v8/src/codegen/safepoint-table.h b/deps/v8/src/codegen/safepoint-table.h index 9efdbfa784..623b524698 100644 --- a/deps/v8/src/codegen/safepoint-table.h +++ b/deps/v8/src/codegen/safepoint-table.h @@ -50,6 +50,18 @@ class SafepointEntry { return deopt_index_; } + uint32_t register_bits() const { + // The register bits use the same field as the deopt_index_. + DCHECK(is_valid()); + return deopt_index_; + } + + bool has_register_bits() const { + // The register bits use the same field as the deopt_index_. + DCHECK(is_valid()); + return deopt_index_ != kNoDeoptIndex; + } + bool has_deoptimization_index() const { DCHECK(is_valid()); return deopt_index_ != kNoDeoptIndex; @@ -61,7 +73,7 @@ class SafepointEntry { } private: - unsigned deopt_index_; + uint32_t deopt_index_; uint8_t* bits_; // It needs to be an integer as it is -1 for eager deoptimizations. int trampoline_pc_; @@ -69,8 +81,12 @@ class SafepointEntry { class SafepointTable { public: - explicit SafepointTable(Code code); + // The isolate and pc arguments are used for figuring out whether pc + // belongs to the embedded or un-embedded code blob. + explicit SafepointTable(Isolate* isolate, Address pc, Code code); +#if V8_ENABLE_WEBASSEMBLY explicit SafepointTable(const wasm::WasmCode* code); +#endif // V8_ENABLE_WEBASSEMBLY SafepointTable(const SafepointTable&) = delete; SafepointTable& operator=(const SafepointTable&) = delete; @@ -170,11 +186,20 @@ class Safepoint { public: static const int kNoDeoptimizationIndex = SafepointEntry::kNoDeoptIndex; - void DefinePointerSlot(int index) { indexes_->push_back(index); } + void DefinePointerSlot(int index) { stack_indexes_->push_back(index); } + + void DefineRegister(int reg_code) { + // Make sure the recorded index is always less than 31, so that we don't + // generate {kNoDeoptimizationIndex} by accident. + DCHECK_LT(reg_code, 31); + *register_indexes_ |= 1u << reg_code; + } private: - explicit Safepoint(ZoneChunkList<int>* indexes) : indexes_(indexes) {} - ZoneChunkList<int>* const indexes_; + Safepoint(ZoneChunkList<int>* stack_indexes, uint32_t* register_indexes) + : stack_indexes_(stack_indexes), register_indexes_(register_indexes) {} + ZoneChunkList<int>* const stack_indexes_; + uint32_t* register_indexes_; friend class SafepointTableBuilder; }; @@ -211,13 +236,15 @@ class SafepointTableBuilder { unsigned pc; unsigned deopt_index; int trampoline; - ZoneChunkList<int>* indexes; + ZoneChunkList<int>* stack_indexes; + uint32_t register_indexes; DeoptimizationInfo(Zone* zone, unsigned pc) : pc(pc), deopt_index(Safepoint::kNoDeoptimizationIndex), trampoline(-1), - indexes(zone->New<ZoneChunkList<int>>( - zone, ZoneChunkList<int>::StartMode::kSmall)) {} + stack_indexes(zone->New<ZoneChunkList<int>>( + zone, ZoneChunkList<int>::StartMode::kSmall)), + register_indexes(0) {} }; // Compares all fields of a {DeoptimizationInfo} except {pc} and {trampoline}. diff --git a/deps/v8/src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.cc b/deps/v8/src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.cc new file mode 100644 index 0000000000..366d1afac9 --- /dev/null +++ b/deps/v8/src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.cc @@ -0,0 +1,403 @@ +// Copyright 2021 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.h" + +#include "src/codegen/assembler.h" +#include "src/codegen/cpu-features.h" + +#if V8_TARGET_ARCH_IA32 +#include "src/codegen/ia32/register-ia32.h" +#elif V8_TARGET_ARCH_X64 +#include "src/codegen/x64/register-x64.h" +#else +#error Unsupported target architecture. +#endif + +namespace v8 { +namespace internal { + +void SharedTurboAssembler::Movapd(XMMRegister dst, XMMRegister src) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vmovapd(dst, src); + } else { + // On SSE, movaps is 1 byte shorter than movapd, and has the same + // behavior. + movaps(dst, src); + } +} + +void SharedTurboAssembler::S128Store32Lane(Operand dst, XMMRegister src, + uint8_t laneidx) { + if (laneidx == 0) { + Movss(dst, src); + } else { + DCHECK_GE(3, laneidx); + Extractps(dst, src, laneidx); + } +} + +void SharedTurboAssembler::I16x8ExtMulLow(XMMRegister dst, XMMRegister src1, + XMMRegister src2, XMMRegister scratch, + bool is_signed) { + is_signed ? Pmovsxbw(scratch, src1) : Pmovzxbw(scratch, src1); + is_signed ? Pmovsxbw(dst, src2) : Pmovzxbw(dst, src2); + Pmullw(dst, scratch); +} + +void SharedTurboAssembler::I16x8ExtMulHighS(XMMRegister dst, XMMRegister src1, + XMMRegister src2, + XMMRegister scratch) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpunpckhbw(scratch, src1, src1); + vpsraw(scratch, scratch, 8); + vpunpckhbw(dst, src2, src2); + vpsraw(dst, dst, 8); + vpmullw(dst, dst, scratch); + } else { + if (dst != src1) { + movaps(dst, src1); + } + movaps(scratch, src2); + punpckhbw(dst, dst); + psraw(dst, 8); + punpckhbw(scratch, scratch); + psraw(scratch, 8); + pmullw(dst, scratch); + } +} + +void SharedTurboAssembler::I16x8ExtMulHighU(XMMRegister dst, XMMRegister src1, + XMMRegister src2, + XMMRegister scratch) { + // The logic here is slightly complicated to handle all the cases of register + // aliasing. This allows flexibility for callers in TurboFan and Liftoff. + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + if (src1 == src2) { + vpxor(scratch, scratch, scratch); + vpunpckhbw(dst, src1, scratch); + vpmullw(dst, dst, dst); + } else { + if (dst == src2) { + // We overwrite dst, then use src2, so swap src1 and src2. + std::swap(src1, src2); + } + vpxor(scratch, scratch, scratch); + vpunpckhbw(dst, src1, scratch); + vpunpckhbw(scratch, src2, scratch); + vpmullw(dst, dst, scratch); + } + } else { + if (src1 == src2) { + xorps(scratch, scratch); + if (dst != src1) { + movaps(dst, src1); + } + punpckhbw(dst, scratch); + pmullw(dst, scratch); + } else { + // When dst == src1, nothing special needs to be done. + // When dst == src2, swap src1 and src2, since we overwrite dst. + // When dst is unique, copy src1 to dst first. + if (dst == src2) { + std::swap(src1, src2); + // Now, dst == src1. + } else if (dst != src1) { + // dst != src1 && dst != src2. + movaps(dst, src1); + } + xorps(scratch, scratch); + punpckhbw(dst, scratch); + punpckhbw(scratch, src2); + psrlw(scratch, 8); + pmullw(dst, scratch); + } + } +} + +void SharedTurboAssembler::I16x8SConvertI8x16High(XMMRegister dst, + XMMRegister src) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + // src = |a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p| (high) + // dst = |i|i|j|j|k|k|l|l|m|m|n|n|o|o|p|p| + vpunpckhbw(dst, src, src); + vpsraw(dst, dst, 8); + } else { + CpuFeatureScope sse_scope(this, SSE4_1); + if (dst == src) { + // 2 bytes shorter than pshufd, but has depdency on dst. + movhlps(dst, src); + pmovsxbw(dst, dst); + } else { + // No dependency on dst. + pshufd(dst, src, 0xEE); + pmovsxbw(dst, dst); + } + } +} + +void SharedTurboAssembler::I16x8UConvertI8x16High(XMMRegister dst, + XMMRegister src, + XMMRegister scratch) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + // tmp = |0|0|0|0|0|0|0|0 | 0|0|0|0|0|0|0|0| + // src = |a|b|c|d|e|f|g|h | i|j|k|l|m|n|o|p| + // dst = |0|a|0|b|0|c|0|d | 0|e|0|f|0|g|0|h| + XMMRegister tmp = dst == src ? scratch : dst; + vpxor(tmp, tmp, tmp); + vpunpckhbw(dst, src, tmp); + } else { + CpuFeatureScope sse_scope(this, SSE4_1); + if (dst == src) { + // xorps can be executed on more ports than pshufd. + xorps(scratch, scratch); + punpckhbw(dst, scratch); + } else { + // No dependency on dst. + pshufd(dst, src, 0xEE); + pmovzxbw(dst, dst); + } + } +} + +// 1. Multiply low word into scratch. +// 2. Multiply high word (can be signed or unsigned) into dst. +// 3. Unpack and interleave scratch and dst into dst. +void SharedTurboAssembler::I32x4ExtMul(XMMRegister dst, XMMRegister src1, + XMMRegister src2, XMMRegister scratch, + bool low, bool is_signed) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpmullw(scratch, src1, src2); + is_signed ? vpmulhw(dst, src1, src2) : vpmulhuw(dst, src1, src2); + low ? vpunpcklwd(dst, scratch, dst) : vpunpckhwd(dst, scratch, dst); + } else { + DCHECK_EQ(dst, src1); + movaps(scratch, src1); + pmullw(dst, src2); + is_signed ? pmulhw(scratch, src2) : pmulhuw(scratch, src2); + low ? punpcklwd(dst, scratch) : punpckhwd(dst, scratch); + } +} + +void SharedTurboAssembler::I32x4SConvertI16x8High(XMMRegister dst, + XMMRegister src) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + // src = |a|b|c|d|e|f|g|h| (high) + // dst = |e|e|f|f|g|g|h|h| + vpunpckhwd(dst, src, src); + vpsrad(dst, dst, 16); + } else { + CpuFeatureScope sse_scope(this, SSE4_1); + if (dst == src) { + // 2 bytes shorter than pshufd, but has depdency on dst. + movhlps(dst, src); + pmovsxwd(dst, dst); + } else { + // No dependency on dst. + pshufd(dst, src, 0xEE); + pmovsxwd(dst, dst); + } + } +} + +void SharedTurboAssembler::I32x4UConvertI16x8High(XMMRegister dst, + XMMRegister src, + XMMRegister scratch) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + // scratch = |0|0|0|0|0|0|0|0| + // src = |a|b|c|d|e|f|g|h| + // dst = |0|a|0|b|0|c|0|d| + XMMRegister tmp = dst == src ? scratch : dst; + vpxor(tmp, tmp, tmp); + vpunpckhwd(dst, src, tmp); + } else { + if (dst == src) { + // xorps can be executed on more ports than pshufd. + xorps(scratch, scratch); + punpckhwd(dst, scratch); + } else { + CpuFeatureScope sse_scope(this, SSE4_1); + // No dependency on dst. + pshufd(dst, src, 0xEE); + pmovzxwd(dst, dst); + } + } +} + +void SharedTurboAssembler::I64x2Abs(XMMRegister dst, XMMRegister src, + XMMRegister scratch) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + XMMRegister tmp = dst == src ? scratch : dst; + vpxor(tmp, tmp, tmp); + vpsubq(tmp, tmp, src); + vblendvpd(dst, src, tmp, src); + } else { + CpuFeatureScope sse_scope(this, SSE3); + movshdup(scratch, src); + if (dst != src) { + movaps(dst, src); + } + psrad(scratch, 31); + xorps(dst, scratch); + psubq(dst, scratch); + } +} + +void SharedTurboAssembler::I64x2GtS(XMMRegister dst, XMMRegister src0, + XMMRegister src1, XMMRegister scratch) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpcmpgtq(dst, src0, src1); + } else if (CpuFeatures::IsSupported(SSE4_2)) { + CpuFeatureScope sse_scope(this, SSE4_2); + DCHECK_EQ(dst, src0); + pcmpgtq(dst, src1); + } else { + CpuFeatureScope sse_scope(this, SSE3); + DCHECK_NE(dst, src0); + DCHECK_NE(dst, src1); + movaps(dst, src1); + movaps(scratch, src0); + psubq(dst, src0); + pcmpeqd(scratch, src1); + andps(dst, scratch); + movaps(scratch, src0); + pcmpgtd(scratch, src1); + orps(dst, scratch); + movshdup(dst, dst); + } +} + +void SharedTurboAssembler::I64x2GeS(XMMRegister dst, XMMRegister src0, + XMMRegister src1, XMMRegister scratch) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpcmpgtq(dst, src1, src0); + vpcmpeqd(scratch, scratch, scratch); + vpxor(dst, dst, scratch); + } else if (CpuFeatures::IsSupported(SSE4_2)) { + CpuFeatureScope sse_scope(this, SSE4_2); + DCHECK_NE(dst, src0); + if (dst != src1) { + movaps(dst, src1); + } + pcmpgtq(dst, src0); + pcmpeqd(scratch, scratch); + xorps(dst, scratch); + } else { + CpuFeatureScope sse_scope(this, SSE3); + DCHECK_NE(dst, src0); + DCHECK_NE(dst, src1); + movaps(dst, src0); + movaps(scratch, src1); + psubq(dst, src1); + pcmpeqd(scratch, src0); + andps(dst, scratch); + movaps(scratch, src1); + pcmpgtd(scratch, src0); + orps(dst, scratch); + movshdup(dst, dst); + pcmpeqd(scratch, scratch); + xorps(dst, scratch); + } +} + +// 1. Unpack src0, src1 into even-number elements of scratch. +// 2. Unpack src1, src0 into even-number elements of dst. +// 3. Multiply 1. with 2. +// For non-AVX, use non-destructive pshufd instead of punpckldq/punpckhdq. +void SharedTurboAssembler::I64x2ExtMul(XMMRegister dst, XMMRegister src1, + XMMRegister src2, XMMRegister scratch, + bool low, bool is_signed) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + if (low) { + vpunpckldq(scratch, src1, src1); + vpunpckldq(dst, src2, src2); + } else { + vpunpckhdq(scratch, src1, src1); + vpunpckhdq(dst, src2, src2); + } + if (is_signed) { + vpmuldq(dst, scratch, dst); + } else { + vpmuludq(dst, scratch, dst); + } + } else { + uint8_t mask = low ? 0x50 : 0xFA; + pshufd(scratch, src1, mask); + pshufd(dst, src2, mask); + if (is_signed) { + CpuFeatureScope sse4_scope(this, SSE4_1); + pmuldq(dst, scratch); + } else { + pmuludq(dst, scratch); + } + } +} + +void SharedTurboAssembler::I64x2SConvertI32x4High(XMMRegister dst, + XMMRegister src) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpunpckhqdq(dst, src, src); + vpmovsxdq(dst, dst); + } else { + CpuFeatureScope sse_scope(this, SSE4_1); + if (dst == src) { + movhlps(dst, src); + } else { + pshufd(dst, src, 0xEE); + } + pmovsxdq(dst, dst); + } +} + +void SharedTurboAssembler::I64x2UConvertI32x4High(XMMRegister dst, + XMMRegister src, + XMMRegister scratch) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpxor(scratch, scratch, scratch); + vpunpckhdq(dst, src, scratch); + } else { + if (dst != src) { + movaps(dst, src); + } + xorps(scratch, scratch); + punpckhdq(dst, scratch); + } +} + +void SharedTurboAssembler::S128Select(XMMRegister dst, XMMRegister mask, + XMMRegister src1, XMMRegister src2, + XMMRegister scratch) { + // v128.select = v128.or(v128.and(v1, c), v128.andnot(v2, c)). + // pandn(x, y) = !x & y, so we have to flip the mask and input. + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vpandn(scratch, mask, src2); + vpand(dst, src1, mask); + vpor(dst, dst, scratch); + } else { + DCHECK_EQ(dst, mask); + // Use float ops as they are 1 byte shorter than int ops. + movaps(scratch, mask); + andnps(scratch, src2); + andps(dst, src1); + orps(dst, scratch); + } +} + +} // namespace internal +} // namespace v8 diff --git a/deps/v8/src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.h b/deps/v8/src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.h new file mode 100644 index 0000000000..e2778e472d --- /dev/null +++ b/deps/v8/src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.h @@ -0,0 +1,189 @@ +// Copyright 2021 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef V8_CODEGEN_SHARED_IA32_X64_MACRO_ASSEMBLER_SHARED_IA32_X64_H_ +#define V8_CODEGEN_SHARED_IA32_X64_MACRO_ASSEMBLER_SHARED_IA32_X64_H_ + +#include "src/base/macros.h" +#include "src/codegen/cpu-features.h" +#include "src/codegen/turbo-assembler.h" + +#if V8_TARGET_ARCH_IA32 +#include "src/codegen/ia32/register-ia32.h" +#elif V8_TARGET_ARCH_X64 +#include "src/codegen/x64/register-x64.h" +#else +#error Unsupported target architecture. +#endif + +namespace v8 { +namespace internal { +class Assembler; + +class V8_EXPORT_PRIVATE SharedTurboAssembler : public TurboAssemblerBase { + public: + using TurboAssemblerBase::TurboAssemblerBase; + + void Movapd(XMMRegister dst, XMMRegister src); + + template <typename Dst, typename Src> + void Movdqu(Dst dst, Src src) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + vmovdqu(dst, src); + } else { + // movups is 1 byte shorter than movdqu. On most SSE systems, this incurs + // no delay moving between integer and floating-point domain. + movups(dst, src); + } + } + + template <typename Dst, typename... Args> + struct AvxHelper { + Assembler* assm; + base::Optional<CpuFeature> feature = base::nullopt; + // Call a method where the AVX version expects the dst argument to be + // duplicated. + template <void (Assembler::*avx)(Dst, Dst, Args...), + void (Assembler::*no_avx)(Dst, Args...)> + void emit(Dst dst, Args... args) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope scope(assm, AVX); + (assm->*avx)(dst, dst, args...); + } else if (feature.has_value()) { + DCHECK(CpuFeatures::IsSupported(*feature)); + CpuFeatureScope scope(assm, *feature); + (assm->*no_avx)(dst, args...); + } else { + (assm->*no_avx)(dst, args...); + } + } + + // Call a method where the AVX version expects no duplicated dst argument. + template <void (Assembler::*avx)(Dst, Args...), + void (Assembler::*no_avx)(Dst, Args...)> + void emit(Dst dst, Args... args) { + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope scope(assm, AVX); + (assm->*avx)(dst, args...); + } else if (feature.has_value()) { + DCHECK(CpuFeatures::IsSupported(*feature)); + CpuFeatureScope scope(assm, *feature); + (assm->*no_avx)(dst, args...); + } else { + (assm->*no_avx)(dst, args...); + } + } + }; + +#define AVX_OP(macro_name, name) \ + template <typename Dst, typename... Args> \ + void macro_name(Dst dst, Args... args) { \ + AvxHelper<Dst, Args...>{this} \ + .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ + } + +#define AVX_OP_SSE3(macro_name, name) \ + template <typename Dst, typename... Args> \ + void macro_name(Dst dst, Args... args) { \ + AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE3)} \ + .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ + } + +#define AVX_OP_SSSE3(macro_name, name) \ + template <typename Dst, typename... Args> \ + void macro_name(Dst dst, Args... args) { \ + AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSSE3)} \ + .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ + } + +#define AVX_OP_SSE4_1(macro_name, name) \ + template <typename Dst, typename... Args> \ + void macro_name(Dst dst, Args... args) { \ + AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE4_1)} \ + .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ + } + +#define AVX_OP_SSE4_2(macro_name, name) \ + template <typename Dst, typename... Args> \ + void macro_name(Dst dst, Args... args) { \ + AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE4_2)} \ + .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ + } + + AVX_OP(Cvtdq2pd, cvtdq2pd) + AVX_OP(Cvtdq2ps, cvtdq2ps) + AVX_OP(Cvtps2pd, cvtps2pd) + AVX_OP(Cvtpd2ps, cvtpd2ps) + AVX_OP(Cvttps2dq, cvttps2dq) + AVX_OP(Movaps, movaps) + AVX_OP(Movd, movd) + AVX_OP(Movhps, movhps) + AVX_OP(Movlps, movlps) + AVX_OP(Movmskpd, movmskpd) + AVX_OP(Movmskps, movmskps) + AVX_OP(Movss, movss) + AVX_OP(Movsd, movsd) + AVX_OP(Movupd, movupd) + AVX_OP(Movups, movups) + AVX_OP(Pmovmskb, pmovmskb) + AVX_OP(Pmullw, pmullw) + AVX_OP(Pshuflw, pshuflw) + AVX_OP(Pshufhw, pshufhw) + AVX_OP(Pshufd, pshufd) + AVX_OP(Rcpps, rcpps) + AVX_OP(Rsqrtps, rsqrtps) + AVX_OP(Sqrtps, sqrtps) + AVX_OP(Sqrtpd, sqrtpd) + AVX_OP_SSE3(Movddup, movddup) + AVX_OP_SSE3(Movshdup, movshdup) + AVX_OP_SSSE3(Pabsb, pabsb) + AVX_OP_SSSE3(Pabsw, pabsw) + AVX_OP_SSSE3(Pabsd, pabsd) + AVX_OP_SSE4_1(Extractps, extractps) + AVX_OP_SSE4_1(Pextrb, pextrb) + AVX_OP_SSE4_1(Pextrw, pextrw) + AVX_OP_SSE4_1(Pmovsxbw, pmovsxbw) + AVX_OP_SSE4_1(Pmovsxwd, pmovsxwd) + AVX_OP_SSE4_1(Pmovsxdq, pmovsxdq) + AVX_OP_SSE4_1(Pmovzxbw, pmovzxbw) + AVX_OP_SSE4_1(Pmovzxwd, pmovzxwd) + AVX_OP_SSE4_1(Pmovzxdq, pmovzxdq) + AVX_OP_SSE4_1(Ptest, ptest) + AVX_OP_SSE4_1(Roundps, roundps) + AVX_OP_SSE4_1(Roundpd, roundpd) + + void S128Store32Lane(Operand dst, XMMRegister src, uint8_t laneidx); + void I16x8ExtMulLow(XMMRegister dst, XMMRegister src1, XMMRegister src2, + XMMRegister scrat, bool is_signed); + void I16x8ExtMulHighS(XMMRegister dst, XMMRegister src1, XMMRegister src2, + XMMRegister scratch); + void I16x8ExtMulHighU(XMMRegister dst, XMMRegister src1, XMMRegister src2, + XMMRegister scratch); + void I16x8SConvertI8x16High(XMMRegister dst, XMMRegister src); + void I16x8UConvertI8x16High(XMMRegister dst, XMMRegister src, + XMMRegister scratch); + // Requires that dst == src1 if AVX is not supported. + void I32x4ExtMul(XMMRegister dst, XMMRegister src1, XMMRegister src2, + XMMRegister scratch, bool low, bool is_signed); + void I32x4SConvertI16x8High(XMMRegister dst, XMMRegister src); + void I32x4UConvertI16x8High(XMMRegister dst, XMMRegister src, + XMMRegister scratch); + void I64x2Abs(XMMRegister dst, XMMRegister src, XMMRegister scratch); + void I64x2GtS(XMMRegister dst, XMMRegister src0, XMMRegister src1, + XMMRegister scratch); + void I64x2GeS(XMMRegister dst, XMMRegister src0, XMMRegister src1, + XMMRegister scratch); + void I64x2ExtMul(XMMRegister dst, XMMRegister src1, XMMRegister src2, + XMMRegister scratch, bool low, bool is_signed); + void I64x2SConvertI32x4High(XMMRegister dst, XMMRegister src); + void I64x2UConvertI32x4High(XMMRegister dst, XMMRegister src, + XMMRegister scratch); + // Requires dst == mask when AVX is not supported. + void S128Select(XMMRegister dst, XMMRegister mask, XMMRegister src1, + XMMRegister src2, XMMRegister scratch); +}; +} // namespace internal +} // namespace v8 +#endif // V8_CODEGEN_SHARED_IA32_X64_MACRO_ASSEMBLER_SHARED_IA32_X64_H_ diff --git a/deps/v8/src/codegen/signature.h b/deps/v8/src/codegen/signature.h index bba3a1b13d..2c4ca3e0d9 100644 --- a/deps/v8/src/codegen/signature.h +++ b/deps/v8/src/codegen/signature.h @@ -124,6 +124,60 @@ size_t hash_value(const Signature<T>& sig) { return hash; } +template <typename T, size_t kNumReturns = 0, size_t kNumParams = 0> +class FixedSizeSignature : public Signature<T> { + public: + // Add return types to this signature (only allowed if there are none yet). + template <typename... ReturnTypes> + auto Returns(ReturnTypes... return_types) const { + static_assert(kNumReturns == 0, "Please specify all return types at once"); + return FixedSizeSignature<T, sizeof...(ReturnTypes), kNumParams>{ + std::initializer_list<T>{return_types...}.begin(), reps_}; + } + + // Add parameters to this signature (only allowed if there are none yet). + template <typename... ParamTypes> + auto Params(ParamTypes... param_types) const { + static_assert(kNumParams == 0, "Please specify all parameters at once"); + return FixedSizeSignature<T, kNumReturns, sizeof...(ParamTypes)>{ + reps_, std::initializer_list<T>{param_types...}.begin()}; + } + + private: + // Other template instantiations can call the private constructor. + template <typename T2, size_t kNumReturns2, size_t kNumParams2> + friend class FixedSizeSignature; + + FixedSizeSignature(const T* returns, const T* params) + : Signature<T>(kNumReturns, kNumParams, reps_) { + std::copy(returns, returns + kNumReturns, reps_); + std::copy(params, params + kNumParams, reps_ + kNumReturns); + } + + T reps_[kNumReturns + kNumParams]; +}; + +// Specialization for zero-sized signatures. +template <typename T> +class FixedSizeSignature<T, 0, 0> : public Signature<T> { + public: + constexpr FixedSizeSignature() : Signature<T>(0, 0, nullptr) {} + + // Add return types. + template <typename... ReturnTypes> + static auto Returns(ReturnTypes... return_types) { + return FixedSizeSignature<T, sizeof...(ReturnTypes), 0>{ + std::initializer_list<T>{return_types...}.begin(), nullptr}; + } + + // Add parameters. + template <typename... ParamTypes> + static auto Params(ParamTypes... param_types) { + return FixedSizeSignature<T, 0, sizeof...(ParamTypes)>{ + nullptr, std::initializer_list<T>{param_types...}.begin()}; + } +}; + } // namespace internal } // namespace v8 diff --git a/deps/v8/src/codegen/tnode.h b/deps/v8/src/codegen/tnode.h index a6cfc6983a..3a56fda9d1 100644 --- a/deps/v8/src/codegen/tnode.h +++ b/deps/v8/src/codegen/tnode.h @@ -110,6 +110,16 @@ struct BoolT : Word32T {}; template <class T1, class T2> struct PairT {}; +struct Simd128T : UntaggedT { + static const MachineRepresentation kMachineRepresentation = + MachineRepresentation::kSimd128; + static constexpr MachineType kMachineType = MachineType::Simd128(); +}; + +struct I8x16T : Simd128T {}; +struct I16x8T : Simd128T {}; +struct I32x2T : Simd128T {}; + inline constexpr MachineType CommonMachineType(MachineType type1, MachineType type2) { return (type1 == type2) ? type1 @@ -347,16 +357,12 @@ class TNode { return *this; } - bool is_null() const { return node_ == nullptr; } - operator compiler::Node*() const { return node_; } static TNode UncheckedCast(compiler::Node* node) { return TNode(node); } - protected: - explicit TNode(compiler::Node* node) : node_(node) { LazyTemplateChecks(); } - private: + explicit TNode(compiler::Node* node) : node_(node) { LazyTemplateChecks(); } // These checks shouldn't be checked before TNode is actually used. void LazyTemplateChecks() { static_assert(is_valid_type_tag<T>::value, "invalid type tag"); @@ -365,21 +371,6 @@ class TNode { compiler::Node* node_; }; -// SloppyTNode<T> is a variant of TNode<T> and allows implicit casts from -// Node*. It is intended for function arguments as long as some call sites -// still use untyped Node* arguments. -// TODO(turbofan): Delete this class once transition is finished. -template <class T> -class SloppyTNode : public TNode<T> { - public: - SloppyTNode(compiler::Node* node) // NOLINT(runtime/explicit) - : TNode<T>(node) {} - template <class U, typename std::enable_if<is_subtype<U, T>::value, - int>::type = 0> - SloppyTNode(const TNode<U>& other) // NOLINT(runtime/explicit) - : TNode<T>(other) {} -}; - } // namespace internal } // namespace v8 diff --git a/deps/v8/src/codegen/x64/assembler-x64-inl.h b/deps/v8/src/codegen/x64/assembler-x64-inl.h index 836566a1ac..02879ba354 100644 --- a/deps/v8/src/codegen/x64/assembler-x64-inl.h +++ b/deps/v8/src/codegen/x64/assembler-x64-inl.h @@ -17,12 +17,6 @@ namespace internal { bool CpuFeatures::SupportsOptimizer() { return true; } -bool CpuFeatures::SupportsWasmSimd128() { - if (IsSupported(SSE4_1)) return true; - if (FLAG_wasm_simd_ssse3_codegen && IsSupported(SSSE3)) return true; - return false; -} - // ----------------------------------------------------------------------------- // Implementation of Assembler @@ -41,8 +35,10 @@ void Assembler::emitw(uint16_t x) { pc_ += sizeof(uint16_t); } +// TODO(ishell): Rename accordingly once RUNTIME_ENTRY is renamed. void Assembler::emit_runtime_entry(Address entry, RelocInfo::Mode rmode) { DCHECK(RelocInfo::IsRuntimeEntry(rmode)); + DCHECK_NE(options().code_range_start, 0); RecordRelocInfo(rmode); emitl(static_cast<uint32_t>(entry - options().code_range_start)); } diff --git a/deps/v8/src/codegen/x64/assembler-x64.cc b/deps/v8/src/codegen/x64/assembler-x64.cc index 18330a9126..032f7eb13d 100644 --- a/deps/v8/src/codegen/x64/assembler-x64.cc +++ b/deps/v8/src/codegen/x64/assembler-x64.cc @@ -71,6 +71,14 @@ bool OSHasAVXSupport() { } // namespace +bool CpuFeatures::SupportsWasmSimd128() { +#if V8_ENABLE_WEBASSEMBLY + if (IsSupported(SSE4_1)) return true; + if (FLAG_wasm_simd_ssse3_codegen && IsSupported(SSSE3)) return true; +#endif // V8_ENABLE_WEBASSEMBLY + return false; +} + void CpuFeatures::ProbeImpl(bool cross_compile) { base::CPU cpu; CHECK(cpu.has_sse2()); // SSE2 support is mandatory. @@ -79,19 +87,9 @@ void CpuFeatures::ProbeImpl(bool cross_compile) { // Only use statically determined features for cross compile (snapshot). if (cross_compile) return; - if (cpu.has_sse42() && FLAG_enable_sse4_2) supported_ |= 1u << SSE4_2; - if (cpu.has_sse41() && FLAG_enable_sse4_1) { - supported_ |= 1u << SSE4_1; - supported_ |= 1u << SSSE3; - } - if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3; - if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3; - // SAHF is not generally available in long mode. - if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF; - if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() && - OSHasAVXSupport()) { - supported_ |= 1u << AVX; - } + // To deal with any combination of flags (e.g. --no-enable-sse4-1 + // --enable-sse-4-2), we start checking from the "highest" supported + // extension, for each extension, enable if newer extension is supported. if (cpu.has_avx2() && FLAG_enable_avx2 && IsSupported(AVX)) { supported_ |= 1u << AVX2; } @@ -99,6 +97,24 @@ void CpuFeatures::ProbeImpl(bool cross_compile) { OSHasAVXSupport()) { supported_ |= 1u << FMA3; } + if ((cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() && + OSHasAVXSupport()) || + IsSupported(AVX2) || IsSupported(FMA3)) { + supported_ |= 1u << AVX; + } + if ((cpu.has_sse42() && FLAG_enable_sse4_2) || IsSupported(AVX)) { + supported_ |= 1u << SSE4_2; + } + if ((cpu.has_sse41() && FLAG_enable_sse4_1) || IsSupported(SSE4_2)) { + supported_ |= 1u << SSE4_1; + } + if ((cpu.has_ssse3() && FLAG_enable_ssse3) || IsSupported(SSE4_1)) { + supported_ |= 1u << SSSE3; + } + if ((cpu.has_sse3() && FLAG_enable_sse3) || IsSupported(SSSE3)) + supported_ |= 1u << SSE3; + // SAHF is not generally available in long mode. + if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF; if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1; if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2; if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT; @@ -1194,16 +1210,6 @@ void Assembler::cpuid() { emit(0xA2); } -void Assembler::prefetch(Operand src, int level) { - DCHECK(is_uint2(level)); - EnsureSpace ensure_space(this); - emit(0x0F); - emit(0x18); - // Emit hint number in Reg position of RegR/M. - XMMRegister code = XMMRegister::from_code(level); - emit_sse_operand(code, src); -} - void Assembler::cqo() { EnsureSpace ensure_space(this); emit_rex_64(); @@ -1439,6 +1445,14 @@ void Assembler::j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode) { emitl(code_target_index); } +void Assembler::jmp(Address entry, RelocInfo::Mode rmode) { + DCHECK(RelocInfo::IsRuntimeEntry(rmode)); + EnsureSpace ensure_space(this); + // 1110 1001 #32-bit disp. + emit(0xE9); + emit_runtime_entry(entry, rmode); +} + void Assembler::jmp_rel(int32_t offset) { EnsureSpace ensure_space(this); // The offset is encoded relative to the next instruction. diff --git a/deps/v8/src/codegen/x64/assembler-x64.h b/deps/v8/src/codegen/x64/assembler-x64.h index c1dc4a3db1..97e18ed8fe 100644 --- a/deps/v8/src/codegen/x64/assembler-x64.h +++ b/deps/v8/src/codegen/x64/assembler-x64.h @@ -786,7 +786,6 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { void ret(int imm16); void ud2(); void setcc(Condition cc, Register reg); - void prefetch(Operand src, int level); void pblendw(XMMRegister dst, Operand src, uint8_t mask); void pblendw(XMMRegister dst, XMMRegister src, uint8_t mask); @@ -833,6 +832,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { // Unconditional jump to L void jmp(Label* L, Label::Distance distance = Label::kFar); void jmp(Handle<Code> target, RelocInfo::Mode rmode); + void jmp(Address entry, RelocInfo::Mode rmode); // Jump near absolute indirect (r64) void jmp(Register adr); diff --git a/deps/v8/src/codegen/x64/macro-assembler-x64.cc b/deps/v8/src/codegen/x64/macro-assembler-x64.cc index b91e8319ac..cb254370b2 100644 --- a/deps/v8/src/codegen/x64/macro-assembler-x64.cc +++ b/deps/v8/src/codegen/x64/macro-assembler-x64.cc @@ -10,6 +10,7 @@ #include "src/base/utils/random-number-generator.h" #include "src/codegen/callable.h" #include "src/codegen/code-factory.h" +#include "src/codegen/cpu-features.h" #include "src/codegen/external-reference-table.h" #include "src/codegen/macro-assembler.h" #include "src/codegen/register-configuration.h" @@ -269,8 +270,8 @@ void TurboAssembler::StoreTaggedField(Operand dst_field_operand, void TurboAssembler::StoreTaggedSignedField(Operand dst_field_operand, Smi value) { if (SmiValuesAre32Bits()) { - movl(Operand(dst_field_operand, kSmiShift / kBitsPerByte), - Immediate(value.value())); + Move(kScratchRegister, value); + movq(dst_field_operand, kScratchRegister); } else { StoreTaggedField(dst_field_operand, Immediate(value)); } @@ -287,7 +288,7 @@ void TurboAssembler::DecompressTaggedPointer(Register destination, Operand field_operand) { RecordComment("[ DecompressTaggedPointer"); movl(destination, field_operand); - addq(destination, kRootRegister); + addq(destination, kPointerCageBaseRegister); RecordComment("]"); } @@ -295,7 +296,7 @@ void TurboAssembler::DecompressTaggedPointer(Register destination, Register source) { RecordComment("[ DecompressTaggedPointer"); movl(destination, source); - addq(destination, kRootRegister); + addq(destination, kPointerCageBaseRegister); RecordComment("]"); } @@ -303,7 +304,7 @@ void TurboAssembler::DecompressAnyTagged(Register destination, Operand field_operand) { RecordComment("[ DecompressAnyTagged"); movl(destination, field_operand); - addq(destination, kRootRegister); + addq(destination, kPointerCageBaseRegister); RecordComment("]"); } @@ -738,35 +739,6 @@ void TurboAssembler::Movdqa(XMMRegister dst, XMMRegister src) { } } -void TurboAssembler::Movapd(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vmovapd(dst, src); - } else { - // On SSE, movaps is 1 byte shorter than movapd, and has the same behavior. - movaps(dst, src); - } -} - -template <typename Dst, typename Src> -void TurboAssembler::Movdqu(Dst dst, Src src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vmovdqu(dst, src); - } else { - // movups is 1 byte shorter than movdqu. On most SSE systems, this incurs - // no delay moving between integer and floating-point domain. - movups(dst, src); - } -} - -template void TurboAssembler::Movdqu<XMMRegister, Operand>(XMMRegister dst, - Operand src); -template void TurboAssembler::Movdqu<Operand, XMMRegister>(Operand dst, - XMMRegister src); -template void TurboAssembler::Movdqu<XMMRegister, XMMRegister>(XMMRegister dst, - XMMRegister src); - void TurboAssembler::Cvtss2sd(XMMRegister dst, XMMRegister src) { if (CpuFeatures::IsSupported(AVX)) { CpuFeatureScope scope(this, AVX); @@ -1619,14 +1591,7 @@ void TurboAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode, if (cc == never) return; j(NegateCondition(cc), &skip, Label::kNear); } - // Inline the trampoline. - RecordCommentForOffHeapTrampoline(builtin_index); - CHECK_NE(builtin_index, Builtins::kNoBuiltinId); - EmbeddedData d = EmbeddedData::FromBlob(); - Address entry = d.InstructionStartOfBuiltin(builtin_index); - Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET); - jmp(kScratchRegister); - if (FLAG_code_comments) RecordComment("]"); + TailCallBuiltin(builtin_index); bind(&skip); return; } @@ -1705,10 +1670,17 @@ void TurboAssembler::CallBuiltin(int builtin_index) { DCHECK(Builtins::IsBuiltinId(builtin_index)); RecordCommentForOffHeapTrampoline(builtin_index); CHECK_NE(builtin_index, Builtins::kNoBuiltinId); - EmbeddedData d = EmbeddedData::FromBlob(); - Address entry = d.InstructionStartOfBuiltin(builtin_index); - Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET); - call(kScratchRegister); + if (options().short_builtin_calls) { + EmbeddedData d = EmbeddedData::FromBlob(isolate()); + Address entry = d.InstructionStartOfBuiltin(builtin_index); + call(entry, RelocInfo::RUNTIME_ENTRY); + + } else { + EmbeddedData d = EmbeddedData::FromBlob(); + Address entry = d.InstructionStartOfBuiltin(builtin_index); + Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET); + call(kScratchRegister); + } if (FLAG_code_comments) RecordComment("]"); } @@ -1716,10 +1688,16 @@ void TurboAssembler::TailCallBuiltin(int builtin_index) { DCHECK(Builtins::IsBuiltinId(builtin_index)); RecordCommentForOffHeapTrampoline(builtin_index); CHECK_NE(builtin_index, Builtins::kNoBuiltinId); - EmbeddedData d = EmbeddedData::FromBlob(); - Address entry = d.InstructionStartOfBuiltin(builtin_index); - Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET); - jmp(kScratchRegister); + if (options().short_builtin_calls) { + EmbeddedData d = EmbeddedData::FromBlob(isolate()); + Address entry = d.InstructionStartOfBuiltin(builtin_index); + jmp(entry, RelocInfo::RUNTIME_ENTRY); + + } else { + EmbeddedData d = EmbeddedData::FromBlob(); + Address entry = d.InstructionStartOfBuiltin(builtin_index); + Jump(entry, RelocInfo::OFF_HEAP_TARGET); + } if (FLAG_code_comments) RecordComment("]"); } @@ -1936,7 +1914,7 @@ void PinsrHelper(Assembler* assm, AvxFn<Src> avx, NoAvxFn<Src> noavx, } if (dst != src1) { - assm->movdqu(dst, src1); + assm->movaps(dst, src1); } if (feature.has_value()) { DCHECK(CpuFeatures::IsSupported(*feature)); @@ -2108,7 +2086,7 @@ void TurboAssembler::Pshufb(XMMRegister dst, XMMRegister src, // Make sure these are different so that we won't overwrite mask. DCHECK_NE(dst, mask); if (dst != src) { - movapd(dst, src); + movaps(dst, src); } CpuFeatureScope sse_scope(this, SSSE3); pshufb(dst, mask); @@ -2129,189 +2107,6 @@ void TurboAssembler::Pmulhrsw(XMMRegister dst, XMMRegister src1, } } -void TurboAssembler::I32x4SConvertI16x8High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - // src = |a|b|c|d|e|f|g|h| (high) - // dst = |e|e|f|f|g|g|h|h| - vpunpckhwd(dst, src, src); - vpsrad(dst, dst, 16); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - if (dst == src) { - // 2 bytes shorter than pshufd, but has depdency on dst. - movhlps(dst, src); - pmovsxwd(dst, dst); - } else { - // No dependency on dst. - pshufd(dst, src, 0xEE); - pmovsxwd(dst, dst); - } - } -} - -void TurboAssembler::I32x4UConvertI16x8High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - // scratch = |0|0|0|0|0|0|0|0| - // src = |a|b|c|d|e|f|g|h| - // dst = |0|a|0|b|0|c|0|d| - XMMRegister scratch = dst == src ? kScratchDoubleReg : dst; - vpxor(scratch, scratch, scratch); - vpunpckhwd(dst, src, scratch); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - if (dst == src) { - // xorps can be executed on more ports than pshufd. - xorps(kScratchDoubleReg, kScratchDoubleReg); - punpckhwd(dst, kScratchDoubleReg); - } else { - // No dependency on dst. - pshufd(dst, src, 0xEE); - pmovzxwd(dst, dst); - } - } -} - -void TurboAssembler::I16x8SConvertI8x16High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - // src = |a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p| (high) - // dst = |i|i|j|j|k|k|l|l|m|m|n|n|o|o|p|p| - vpunpckhbw(dst, src, src); - vpsraw(dst, dst, 8); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - if (dst == src) { - // 2 bytes shorter than pshufd, but has depdency on dst. - movhlps(dst, src); - pmovsxbw(dst, dst); - } else { - // No dependency on dst. - pshufd(dst, src, 0xEE); - pmovsxbw(dst, dst); - } - } -} - -void TurboAssembler::I16x8UConvertI8x16High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - // scratch = |0|0|0|0|0|0|0|0 | 0|0|0|0|0|0|0|0| - // src = |a|b|c|d|e|f|g|h | i|j|k|l|m|n|o|p| - // dst = |0|a|0|b|0|c|0|d | 0|e|0|f|0|g|0|h| - XMMRegister scratch = dst == src ? kScratchDoubleReg : dst; - vpxor(scratch, scratch, scratch); - vpunpckhbw(dst, src, scratch); - } else { - if (dst == src) { - // xorps can be executed on more ports than pshufd. - xorps(kScratchDoubleReg, kScratchDoubleReg); - punpckhbw(dst, kScratchDoubleReg); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - // No dependency on dst. - pshufd(dst, src, 0xEE); - pmovzxbw(dst, dst); - } - } -} - -void TurboAssembler::I64x2SConvertI32x4High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpunpckhqdq(dst, src, src); - vpmovsxdq(dst, dst); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - pshufd(dst, src, 0xEE); - pmovsxdq(dst, dst); - } -} - -void TurboAssembler::I64x2UConvertI32x4High(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpxor(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); - vpunpckhdq(dst, src, kScratchDoubleReg); - } else { - CpuFeatureScope sse_scope(this, SSE4_1); - pshufd(dst, src, 0xEE); - pmovzxdq(dst, dst); - } -} - -// 1. Unpack src0, src0 into even-number elements of scratch. -// 2. Unpack src1, src1 into even-number elements of dst. -// 3. Multiply 1. with 2. -// For non-AVX, use non-destructive pshufd instead of punpckldq/punpckhdq. -void TurboAssembler::I64x2ExtMul(XMMRegister dst, XMMRegister src1, - XMMRegister src2, bool low, bool is_signed) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - if (low) { - vpunpckldq(kScratchDoubleReg, src1, src1); - vpunpckldq(dst, src2, src2); - } else { - vpunpckhdq(kScratchDoubleReg, src1, src1); - vpunpckhdq(dst, src2, src2); - } - if (is_signed) { - vpmuldq(dst, kScratchDoubleReg, dst); - } else { - vpmuludq(dst, kScratchDoubleReg, dst); - } - } else { - uint8_t mask = low ? 0x50 : 0xFA; - pshufd(kScratchDoubleReg, src1, mask); - pshufd(dst, src2, mask); - if (is_signed) { - CpuFeatureScope avx_scope(this, SSE4_1); - pmuldq(dst, kScratchDoubleReg); - } else { - pmuludq(dst, kScratchDoubleReg); - } - } -} - -// 1. Multiply low word into scratch. -// 2. Multiply high word (can be signed or unsigned) into dst. -// 3. Unpack and interleave scratch and dst into dst. -void TurboAssembler::I32x4ExtMul(XMMRegister dst, XMMRegister src1, - XMMRegister src2, bool low, bool is_signed) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpmullw(kScratchDoubleReg, src1, src2); - is_signed ? vpmulhw(dst, src1, src2) : vpmulhuw(dst, src1, src2); - low ? vpunpcklwd(dst, kScratchDoubleReg, dst) - : vpunpckhwd(dst, kScratchDoubleReg, dst); - } else { - DCHECK_EQ(dst, src1); - movdqu(kScratchDoubleReg, src1); - pmullw(dst, src2); - is_signed ? pmulhw(kScratchDoubleReg, src2) - : pmulhuw(kScratchDoubleReg, src2); - low ? punpcklwd(dst, kScratchDoubleReg) : punpckhwd(dst, kScratchDoubleReg); - } -} - -void TurboAssembler::I16x8ExtMul(XMMRegister dst, XMMRegister src1, - XMMRegister src2, bool low, bool is_signed) { - if (low) { - is_signed ? Pmovsxbw(kScratchDoubleReg, src1) - : Pmovzxbw(kScratchDoubleReg, src1); - is_signed ? Pmovsxbw(dst, src2) : Pmovzxbw(dst, src2); - Pmullw(dst, kScratchDoubleReg); - } else { - Palignr(kScratchDoubleReg, src1, uint8_t{8}); - is_signed ? Pmovsxbw(kScratchDoubleReg, kScratchDoubleReg) - : Pmovzxbw(kScratchDoubleReg, kScratchDoubleReg); - Palignr(dst, src2, uint8_t{8}); - is_signed ? Pmovsxbw(dst, dst) : Pmovzxbw(dst, dst); - Pmullw(dst, kScratchDoubleReg); - } -} - void TurboAssembler::I16x8Q15MulRSatS(XMMRegister dst, XMMRegister src1, XMMRegister src2) { // k = i16x8.splat(0x8000) @@ -2323,16 +2118,6 @@ void TurboAssembler::I16x8Q15MulRSatS(XMMRegister dst, XMMRegister src1, Pxor(dst, kScratchDoubleReg); } -void TurboAssembler::S128Store32Lane(Operand dst, XMMRegister src, - uint8_t laneidx) { - if (laneidx == 0) { - Movss(dst, src); - } else { - DCHECK_GE(3, laneidx); - Extractps(dst, src, laneidx); - } -} - void TurboAssembler::S128Store64Lane(Operand dst, XMMRegister src, uint8_t laneidx) { if (laneidx == 0) { @@ -2347,6 +2132,7 @@ void TurboAssembler::I8x16Popcnt(XMMRegister dst, XMMRegister src, XMMRegister tmp) { DCHECK_NE(dst, tmp); DCHECK_NE(src, tmp); + DCHECK_NE(kScratchDoubleReg, tmp); if (CpuFeatures::IsSupported(AVX)) { CpuFeatureScope avx_scope(this, AVX); vmovdqa(tmp, ExternalReferenceAsOperand( @@ -2364,8 +2150,8 @@ void TurboAssembler::I8x16Popcnt(XMMRegister dst, XMMRegister src, // PSHUFB instruction, thus use PSHUFB-free divide-and-conquer // algorithm on these processors. ATOM CPU feature captures exactly // the right set of processors. - xorps(tmp, tmp); - pavgb(tmp, src); + movaps(tmp, src); + psrlw(tmp, 1); if (dst != src) { movaps(dst, src); } @@ -2405,6 +2191,10 @@ void TurboAssembler::F64x2ConvertLowI32x4U(XMMRegister dst, XMMRegister src) { // dst = [ src_low, 0x43300000, src_high, 0x4330000 ]; // 0x43300000'00000000 is a special double where the significand bits // precisely represents all uint32 numbers. + if (!CpuFeatures::IsSupported(AVX) && dst != src) { + movaps(dst, src); + src = dst; + } Unpcklps(dst, src, ExternalReferenceAsOperand( ExternalReference:: @@ -2485,82 +2275,6 @@ void TurboAssembler::I32x4TruncSatF64x2UZero(XMMRegister dst, XMMRegister src) { } } -void TurboAssembler::I64x2Abs(XMMRegister dst, XMMRegister src) { - if (CpuFeatures::IsSupported(AVX)) { - XMMRegister tmp = dst == src ? kScratchDoubleReg : dst; - CpuFeatureScope avx_scope(this, AVX); - vpxor(tmp, tmp, tmp); - vpsubq(tmp, tmp, src); - vblendvpd(dst, src, tmp, src); - } else { - CpuFeatureScope sse_scope(this, SSE3); - movshdup(kScratchDoubleReg, src); - if (dst != src) { - movaps(dst, src); - } - psrad(kScratchDoubleReg, 31); - xorps(dst, kScratchDoubleReg); - psubq(dst, kScratchDoubleReg); - } -} - -void TurboAssembler::I64x2GtS(XMMRegister dst, XMMRegister src0, - XMMRegister src1) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpcmpgtq(dst, src0, src1); - } else if (CpuFeatures::IsSupported(SSE4_2)) { - CpuFeatureScope sse_scope(this, SSE4_2); - DCHECK_EQ(dst, src0); - pcmpgtq(dst, src1); - } else { - DCHECK_NE(dst, src0); - DCHECK_NE(dst, src1); - movdqa(dst, src1); - movdqa(kScratchDoubleReg, src0); - psubq(dst, src0); - pcmpeqd(kScratchDoubleReg, src1); - pand(dst, kScratchDoubleReg); - movdqa(kScratchDoubleReg, src0); - pcmpgtd(kScratchDoubleReg, src1); - por(dst, kScratchDoubleReg); - pshufd(dst, dst, 0xF5); - } -} - -void TurboAssembler::I64x2GeS(XMMRegister dst, XMMRegister src0, - XMMRegister src1) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpcmpgtq(dst, src1, src0); - vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg); - vpxor(dst, dst, kScratchDoubleReg); - } else if (CpuFeatures::IsSupported(SSE4_2)) { - CpuFeatureScope sse_scope(this, SSE4_2); - DCHECK_NE(dst, src0); - if (dst != src1) { - movdqa(dst, src1); - } - pcmpgtq(dst, src0); - pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); - pxor(dst, kScratchDoubleReg); - } else { - DCHECK_NE(dst, src0); - DCHECK_NE(dst, src1); - movdqa(dst, src0); - movdqa(kScratchDoubleReg, src1); - psubq(dst, src1); - pcmpeqd(kScratchDoubleReg, src0); - pand(dst, kScratchDoubleReg); - movdqa(kScratchDoubleReg, src1); - pcmpgtd(kScratchDoubleReg, src0); - por(dst, kScratchDoubleReg); - pshufd(dst, dst, 0xF5); - pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); - pxor(dst, kScratchDoubleReg); - } -} - void TurboAssembler::I16x8ExtAddPairwiseI8x16S(XMMRegister dst, XMMRegister src) { // pmaddubsw treats the first operand as unsigned, so the external reference @@ -2586,20 +2300,52 @@ void TurboAssembler::I16x8ExtAddPairwiseI8x16S(XMMRegister dst, void TurboAssembler::I32x4ExtAddPairwiseI16x8U(XMMRegister dst, XMMRegister src) { - // src = |a|b|c|d|e|f|g|h| - // kScratchDoubleReg = i32x4.splat(0x0000FFFF) - Pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); - Psrld(kScratchDoubleReg, byte{16}); - // kScratchDoubleReg =|0|b|0|d|0|f|0|h| - Pand(kScratchDoubleReg, src); - // dst = |0|a|0|c|0|e|0|g| - Psrld(dst, src, byte{16}); - // dst = |a+b|c+d|e+f|g+h| - Paddd(dst, kScratchDoubleReg); + if (CpuFeatures::IsSupported(AVX)) { + CpuFeatureScope avx_scope(this, AVX); + // src = |a|b|c|d|e|f|g|h| (low) + // scratch = |0|a|0|c|0|e|0|g| + vpsrld(kScratchDoubleReg, src, 16); + // dst = |0|b|0|d|0|f|0|h| + vpblendw(dst, src, kScratchDoubleReg, 0xAA); + // dst = |a+b|c+d|e+f|g+h| + vpaddd(dst, kScratchDoubleReg, dst); + } else if (CpuFeatures::IsSupported(SSE4_1)) { + CpuFeatureScope sse_scope(this, SSE4_1); + // There is a potentially better lowering if we get rip-relative constants, + // see https://github.com/WebAssembly/simd/pull/380. + movaps(kScratchDoubleReg, src); + psrld(kScratchDoubleReg, 16); + if (dst != src) { + movaps(dst, src); + } + pblendw(dst, kScratchDoubleReg, 0xAA); + paddd(dst, kScratchDoubleReg); + } else { + // src = |a|b|c|d|e|f|g|h| + // kScratchDoubleReg = i32x4.splat(0x0000FFFF) + pcmpeqd(kScratchDoubleReg, kScratchDoubleReg); + psrld(kScratchDoubleReg, byte{16}); + // kScratchDoubleReg =|0|b|0|d|0|f|0|h| + andps(kScratchDoubleReg, src); + // dst = |0|a|0|c|0|e|0|g| + if (dst != src) { + movaps(dst, src); + } + psrld(dst, byte{16}); + // dst = |a+b|c+d|e+f|g+h| + paddd(dst, kScratchDoubleReg); + } } void TurboAssembler::I8x16Swizzle(XMMRegister dst, XMMRegister src, - XMMRegister mask) { + XMMRegister mask, bool omit_add) { + if (omit_add) { + // We have determined that the indices are immediates, and they are either + // within bounds, or the top bit is set, so we can omit the add. + Pshufb(dst, src, mask); + return; + } + // Out-of-range indices should return 0, add 112 so that any value > 15 // saturates to 128 (top bit set), so pshufb will zero that lane. Operand op = ExternalReferenceAsOperand( @@ -2610,7 +2356,7 @@ void TurboAssembler::I8x16Swizzle(XMMRegister dst, XMMRegister src, vpshufb(dst, src, kScratchDoubleReg); } else { CpuFeatureScope sse_scope(this, SSSE3); - movdqa(kScratchDoubleReg, op); + movaps(kScratchDoubleReg, op); if (dst != src) { movaps(dst, src); } @@ -2644,25 +2390,6 @@ void TurboAssembler::Psrld(XMMRegister dst, XMMRegister src, byte imm8) { } } -void TurboAssembler::S128Select(XMMRegister dst, XMMRegister mask, - XMMRegister src1, XMMRegister src2) { - // v128.select = v128.or(v128.and(v1, c), v128.andnot(v2, c)). - // pandn(x, y) = !x & y, so we have to flip the mask and input. - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope avx_scope(this, AVX); - vpandn(kScratchDoubleReg, mask, src2); - vpand(dst, src1, mask); - vpor(dst, dst, kScratchDoubleReg); - } else { - DCHECK_EQ(dst, mask); - // Use float ops as they are 1 byte shorter than int ops. - movaps(kScratchDoubleReg, mask); - andnps(kScratchDoubleReg, src2); - andps(dst, src1); - orps(dst, kScratchDoubleReg); - } -} - void TurboAssembler::Lzcntl(Register dst, Register src) { if (CpuFeatures::IsSupported(LZCNT)) { CpuFeatureScope scope(this, LZCNT); @@ -2809,12 +2536,6 @@ void TurboAssembler::Popcntq(Register dst, Operand src) { UNREACHABLE(); } -// Order general registers are pushed by Pushad: -// rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r14, r15. -const int - MacroAssembler::kSafepointPushRegisterIndices[Register::kNumRegisters] = { - 0, 1, 2, 3, -1, -1, 4, 5, 6, 7, -1, 8, 9, -1, 10, 11}; - void MacroAssembler::PushStackHandler() { // Adjust this code if not the case. STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize); @@ -3356,7 +3077,7 @@ void TurboAssembler::AllocateStackSpace(int bytes) { } #endif -void MacroAssembler::EnterExitFramePrologue(bool save_rax, +void MacroAssembler::EnterExitFramePrologue(Register saved_rax_reg, StackFrame::Type frame_type) { DCHECK(frame_type == StackFrame::EXIT || frame_type == StackFrame::BUILTIN_EXIT); @@ -3376,8 +3097,8 @@ void MacroAssembler::EnterExitFramePrologue(bool save_rax, Push(Immediate(0)); // Saved entry sp, patched before call. // Save the frame pointer and the context in top. - if (save_rax) { - movq(r14, rax); // Backup rax in callee-save register. + if (saved_rax_reg != no_reg) { + movq(saved_rax_reg, rax); // Backup rax in callee-save register. } Store( @@ -3426,18 +3147,19 @@ void MacroAssembler::EnterExitFrameEpilogue(int arg_stack_space, void MacroAssembler::EnterExitFrame(int arg_stack_space, bool save_doubles, StackFrame::Type frame_type) { - EnterExitFramePrologue(true, frame_type); + Register saved_rax_reg = r12; + EnterExitFramePrologue(saved_rax_reg, frame_type); // Set up argv in callee-saved register r15. It is reused in LeaveExitFrame, // so it must be retained across the C-call. int offset = StandardFrameConstants::kCallerSPOffset - kSystemPointerSize; - leaq(r15, Operand(rbp, r14, times_system_pointer_size, offset)); + leaq(r15, Operand(rbp, saved_rax_reg, times_system_pointer_size, offset)); EnterExitFrameEpilogue(arg_stack_space, save_doubles); } void MacroAssembler::EnterApiExitFrame(int arg_stack_space) { - EnterExitFramePrologue(false, StackFrame::EXIT); + EnterExitFramePrologue(no_reg, StackFrame::EXIT); EnterExitFrameEpilogue(arg_stack_space, false); } @@ -3502,7 +3224,7 @@ static const int kRegisterPassedArguments = 4; static const int kRegisterPassedArguments = 6; #endif -void MacroAssembler::LoadNativeContextSlot(int index, Register dst) { +void MacroAssembler::LoadNativeContextSlot(Register dst, int index) { // Load native context. LoadMap(dst, rsi); LoadTaggedPointerField( diff --git a/deps/v8/src/codegen/x64/macro-assembler-x64.h b/deps/v8/src/codegen/x64/macro-assembler-x64.h index be0b07c17f..40ab1b925c 100644 --- a/deps/v8/src/codegen/x64/macro-assembler-x64.h +++ b/deps/v8/src/codegen/x64/macro-assembler-x64.h @@ -11,6 +11,7 @@ #include "src/base/flags.h" #include "src/codegen/bailout-reason.h" +#include "src/codegen/shared-ia32-x64/macro-assembler-shared-ia32-x64.h" #include "src/codegen/x64/assembler-x64.h" #include "src/common/globals.h" #include "src/objects/contexts.h" @@ -58,99 +59,17 @@ class StackArgumentsAccessor { DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor); }; -class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { +class V8_EXPORT_PRIVATE TurboAssembler : public SharedTurboAssembler { public: - using TurboAssemblerBase::TurboAssemblerBase; - - template <typename Dst, typename... Args> - struct AvxHelper { - Assembler* assm; - base::Optional<CpuFeature> feature = base::nullopt; - // Call a method where the AVX version expects the dst argument to be - // duplicated. - template <void (Assembler::*avx)(Dst, Dst, Args...), - void (Assembler::*no_avx)(Dst, Args...)> - void emit(Dst dst, Args... args) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(assm, AVX); - (assm->*avx)(dst, dst, args...); - } else if (feature.has_value()) { - DCHECK(CpuFeatures::IsSupported(*feature)); - CpuFeatureScope scope(assm, *feature); - (assm->*no_avx)(dst, args...); - } else { - (assm->*no_avx)(dst, args...); - } - } - - // Call a method where the AVX version expects no duplicated dst argument. - template <void (Assembler::*avx)(Dst, Args...), - void (Assembler::*no_avx)(Dst, Args...)> - void emit(Dst dst, Args... args) { - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(assm, AVX); - (assm->*avx)(dst, args...); - } else if (feature.has_value()) { - DCHECK(CpuFeatures::IsSupported(*feature)); - CpuFeatureScope scope(assm, *feature); - (assm->*no_avx)(dst, args...); - } else { - (assm->*no_avx)(dst, args...); - } - } - }; - -#define AVX_OP(macro_name, name) \ - template <typename Dst, typename... Args> \ - void macro_name(Dst dst, Args... args) { \ - AvxHelper<Dst, Args...>{this} \ - .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ - } - -#define AVX_OP_SSE3(macro_name, name) \ - template <typename Dst, typename... Args> \ - void macro_name(Dst dst, Args... args) { \ - AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE3)} \ - .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ - } - -#define AVX_OP_SSSE3(macro_name, name) \ - template <typename Dst, typename... Args> \ - void macro_name(Dst dst, Args... args) { \ - AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSSE3)} \ - .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ - } - -#define AVX_OP_SSE4_1(macro_name, name) \ - template <typename Dst, typename... Args> \ - void macro_name(Dst dst, Args... args) { \ - AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE4_1)} \ - .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ - } -#define AVX_OP_SSE4_2(macro_name, name) \ - template <typename Dst, typename... Args> \ - void macro_name(Dst dst, Args... args) { \ - AvxHelper<Dst, Args...>{this, base::Optional<CpuFeature>(SSE4_2)} \ - .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \ - } + using SharedTurboAssembler::SharedTurboAssembler; AVX_OP(Subsd, subsd) AVX_OP(Divss, divss) AVX_OP(Divsd, divsd) AVX_OP(Orps, orps) AVX_OP(Xorps, xorps) AVX_OP(Xorpd, xorpd) - AVX_OP(Movd, movd) AVX_OP(Movq, movq) - AVX_OP(Movaps, movaps) - AVX_OP(Movups, movups) - AVX_OP(Movmskps, movmskps) - AVX_OP(Movmskpd, movmskpd) - AVX_OP(Pmovmskb, pmovmskb) - AVX_OP(Movss, movss) - AVX_OP(Movsd, movsd) AVX_OP(Movhlps, movhlps) - AVX_OP(Movlps, movlps) - AVX_OP(Movhps, movhps) AVX_OP(Pcmpeqb, pcmpeqb) AVX_OP(Pcmpeqw, pcmpeqw) AVX_OP(Pcmpeqd, pcmpeqd) @@ -182,9 +101,6 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP(Cmpnlepd, cmpnlepd) AVX_OP(Sqrtss, sqrtss) AVX_OP(Sqrtsd, sqrtsd) - AVX_OP(Sqrtps, sqrtps) - AVX_OP(Sqrtpd, sqrtpd) - AVX_OP(Cvttps2dq, cvttps2dq) AVX_OP(Cvttpd2dq, cvttpd2dq) AVX_OP(Ucomiss, ucomiss) AVX_OP(Ucomisd, ucomisd) @@ -218,7 +134,6 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP(Paddusb, paddusb) AVX_OP(Paddusw, paddusw) AVX_OP(Pcmpgtd, pcmpgtd) - AVX_OP(Pmullw, pmullw) AVX_OP(Pmuludq, pmuludq) AVX_OP(Addpd, addpd) AVX_OP(Subpd, subpd) @@ -228,18 +143,10 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP(Divpd, divpd) AVX_OP(Maxps, maxps) AVX_OP(Maxpd, maxpd) - AVX_OP(Cvtdq2ps, cvtdq2ps) - AVX_OP(Cvtdq2pd, cvtdq2pd) - AVX_OP(Cvtpd2ps, cvtpd2ps) - AVX_OP(Cvtps2pd, cvtps2pd) - AVX_OP(Rcpps, rcpps) - AVX_OP(Rsqrtps, rsqrtps) AVX_OP(Addps, addps) AVX_OP(Subps, subps) AVX_OP(Mulps, mulps) AVX_OP(Divps, divps) - AVX_OP(Pshuflw, pshuflw) - AVX_OP(Pshufhw, pshufhw) AVX_OP(Packsswb, packsswb) AVX_OP(Packuswb, packuswb) AVX_OP(Packssdw, packssdw) @@ -251,13 +158,10 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP(Punpckhdq, punpckhdq) AVX_OP(Punpcklqdq, punpcklqdq) AVX_OP(Punpckhqdq, punpckhqdq) - AVX_OP(Pshufd, pshufd) AVX_OP(Cmpps, cmpps) AVX_OP(Cmppd, cmppd) AVX_OP(Movlhps, movlhps) AVX_OP_SSE3(Haddps, haddps) - AVX_OP_SSE3(Movddup, movddup) - AVX_OP_SSE3(Movshdup, movshdup) AVX_OP_SSSE3(Phaddd, phaddd) AVX_OP_SSSE3(Phaddw, phaddw) AVX_OP_SSSE3(Pshufb, pshufb) @@ -265,9 +169,6 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP_SSSE3(Psignw, psignw) AVX_OP_SSSE3(Psignd, psignd) AVX_OP_SSSE3(Palignr, palignr) - AVX_OP_SSSE3(Pabsb, pabsb) - AVX_OP_SSSE3(Pabsw, pabsw) - AVX_OP_SSSE3(Pabsd, pabsd) AVX_OP_SSE4_1(Pcmpeqq, pcmpeqq) AVX_OP_SSE4_1(Packusdw, packusdw) AVX_OP_SSE4_1(Pminsb, pminsb) @@ -279,22 +180,10 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { AVX_OP_SSE4_1(Pmaxuw, pmaxuw) AVX_OP_SSE4_1(Pmaxud, pmaxud) AVX_OP_SSE4_1(Pmulld, pmulld) - AVX_OP_SSE4_1(Extractps, extractps) AVX_OP_SSE4_1(Insertps, insertps) AVX_OP_SSE4_1(Pinsrq, pinsrq) AVX_OP_SSE4_1(Pblendw, pblendw) - AVX_OP_SSE4_1(Ptest, ptest) - AVX_OP_SSE4_1(Pmovsxbw, pmovsxbw) - AVX_OP_SSE4_1(Pmovsxwd, pmovsxwd) - AVX_OP_SSE4_1(Pmovsxdq, pmovsxdq) - AVX_OP_SSE4_1(Pmovzxbw, pmovzxbw) - AVX_OP_SSE4_1(Pmovzxwd, pmovzxwd) - AVX_OP_SSE4_1(Pmovzxdq, pmovzxdq) - AVX_OP_SSE4_1(Pextrb, pextrb) - AVX_OP_SSE4_1(Pextrw, pextrw) AVX_OP_SSE4_1(Pextrq, pextrq) - AVX_OP_SSE4_1(Roundps, roundps) - AVX_OP_SSE4_1(Roundpd, roundpd) AVX_OP_SSE4_1(Roundss, roundss) AVX_OP_SSE4_1(Roundsd, roundsd) AVX_OP_SSE4_2(Pcmpgtq, pcmpgtq) @@ -361,13 +250,9 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { Label* condition_met, Label::Distance condition_met_distance = Label::kFar); - void Movapd(XMMRegister dst, XMMRegister src); void Movdqa(XMMRegister dst, Operand src); void Movdqa(XMMRegister dst, XMMRegister src); - template <typename Dst, typename Src> - void Movdqu(Dst dst, Src src); - void Cvtss2sd(XMMRegister dst, XMMRegister src); void Cvtss2sd(XMMRegister dst, Operand src); void Cvtsd2ss(XMMRegister dst, XMMRegister src); @@ -524,7 +409,15 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { Operand EntryFromBuiltinIndexAsOperand(Builtins::Name builtin_index); Operand EntryFromBuiltinIndexAsOperand(Register builtin_index); void CallBuiltinByIndex(Register builtin_index) override; + void CallBuiltin(Builtins::Name builtin) { + // TODO(11527): drop the int overload in favour of the Builtins::Name one. + return CallBuiltin(static_cast<int>(builtin)); + } void CallBuiltin(int builtin_index); + void TailCallBuiltin(Builtins::Name builtin) { + // TODO(11527): drop the int overload in favour of the Builtins::Name one. + return TailCallBuiltin(static_cast<int>(builtin)); + } void TailCallBuiltin(int builtin_index); void LoadCodeObjectEntry(Register destination, Register code_object) override; @@ -598,28 +491,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { // These Wasm SIMD ops do not have direct lowerings on x64. These // helpers are optimized to produce the fastest and smallest codegen. // Defined here to allow usage on both TurboFan and Liftoff. - void I16x8SConvertI8x16High(XMMRegister dst, XMMRegister src); - void I16x8UConvertI8x16High(XMMRegister dst, XMMRegister src); - void I32x4SConvertI16x8High(XMMRegister dst, XMMRegister src); - void I32x4UConvertI16x8High(XMMRegister dst, XMMRegister src); - void I64x2SConvertI32x4High(XMMRegister dst, XMMRegister src); - void I64x2UConvertI32x4High(XMMRegister dst, XMMRegister src); - - // Requires dst == mask when AVX is not supported. - void S128Select(XMMRegister dst, XMMRegister mask, XMMRegister src1, - XMMRegister src2); - - void I64x2ExtMul(XMMRegister dst, XMMRegister src1, XMMRegister src2, - bool low, bool is_signed); - // Requires that dst == src1 if AVX is not supported. - void I32x4ExtMul(XMMRegister dst, XMMRegister src1, XMMRegister src2, - bool low, bool is_signed); - void I16x8ExtMul(XMMRegister dst, XMMRegister src1, XMMRegister src2, - bool low, bool is_signed); - void I16x8Q15MulRSatS(XMMRegister dst, XMMRegister src1, XMMRegister src2); - void S128Store32Lane(Operand dst, XMMRegister src, uint8_t laneidx); void S128Store64Lane(Operand dst, XMMRegister src, uint8_t laneidx); void I8x16Popcnt(XMMRegister dst, XMMRegister src, XMMRegister tmp); @@ -628,14 +501,11 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void I32x4TruncSatF64x2SZero(XMMRegister dst, XMMRegister src); void I32x4TruncSatF64x2UZero(XMMRegister dst, XMMRegister src); - void I64x2Abs(XMMRegister dst, XMMRegister src); - void I64x2GtS(XMMRegister dst, XMMRegister src0, XMMRegister src1); - void I64x2GeS(XMMRegister dst, XMMRegister src0, XMMRegister src1); - void I16x8ExtAddPairwiseI8x16S(XMMRegister dst, XMMRegister src); void I32x4ExtAddPairwiseI16x8U(XMMRegister dst, XMMRegister src); - void I8x16Swizzle(XMMRegister dst, XMMRegister src, XMMRegister mask); + void I8x16Swizzle(XMMRegister dst, XMMRegister src, XMMRegister mask, + bool omit_add = false); void Abspd(XMMRegister dst); void Negpd(XMMRegister dst); @@ -704,6 +574,9 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase { void InitializeRootRegister() { ExternalReference isolate_root = ExternalReference::isolate_root(isolate()); Move(kRootRegister, isolate_root); +#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE + Move(kPointerCageBaseRegister, isolate_root); +#endif } void SaveRegisters(RegList registers); @@ -1091,11 +964,11 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // Load the global proxy from the current context. void LoadGlobalProxy(Register dst) { - LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst); + LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX); } // Load the native context slot with the current index. - void LoadNativeContextSlot(int index, Register dst); + void LoadNativeContextSlot(Register dst, int index); // --------------------------------------------------------------------------- // Runtime calls @@ -1140,25 +1013,14 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { // In-place weak references. void LoadWeakValue(Register in_out, Label* target_if_cleared); - // --------------------------------------------------------------------------- - // Debugging - - static int SafepointRegisterStackIndex(Register reg) { - return SafepointRegisterStackIndex(reg.code()); - } - private: - // Order general registers are pushed by Pushad. - // rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r12, r14, r15. - static const int kSafepointPushRegisterIndices[Register::kNumRegisters]; - static const int kNumSafepointSavedRegisters = 12; - // Helper functions for generating invokes. void InvokePrologue(Register expected_parameter_count, Register actual_parameter_count, Label* done, InvokeFlag flag); - void EnterExitFramePrologue(bool save_rax, StackFrame::Type frame_type); + void EnterExitFramePrologue(Register saved_rax_reg, + StackFrame::Type frame_type); // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the // stack accessible via StackSpaceOperand. @@ -1166,15 +1028,6 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler { void LeaveExitFrameEpilogue(); - // Compute memory operands for safepoint stack slots. - static int SafepointRegisterStackIndex(int reg_code) { - return kNumSafepointRegisters - kSafepointPushRegisterIndices[reg_code] - 1; - } - - // Needs access to SafepointRegisterStackIndex for compiled frame - // traversal. - friend class CommonFrame; - DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler); }; diff --git a/deps/v8/src/codegen/x64/register-x64.h b/deps/v8/src/codegen/x64/register-x64.h index d9ba5a38b3..9a812d06a1 100644 --- a/deps/v8/src/codegen/x64/register-x64.h +++ b/deps/v8/src/codegen/x64/register-x64.h @@ -29,20 +29,29 @@ namespace internal { V(r14) \ V(r15) -#define ALLOCATABLE_GENERAL_REGISTERS(V) \ - V(rax) \ - V(rbx) \ - V(rdx) \ - V(rcx) \ - V(rsi) \ - V(rdi) \ - V(r8) \ - V(r9) \ - V(r11) \ - V(r12) \ - V(r14) \ +#define ALWAYS_ALLOCATABLE_GENERAL_REGISTERS(V) \ + V(rax) \ + V(rbx) \ + V(rdx) \ + V(rcx) \ + V(rsi) \ + V(rdi) \ + V(r8) \ + V(r9) \ + V(r11) \ + V(r12) \ V(r15) +#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE +#define MAYBE_ALLOCATABLE_GENERAL_REGISTERS(V) +#else +#define MAYBE_ALLOCATABLE_GENERAL_REGISTERS(V) V(r14) +#endif + +#define ALLOCATABLE_GENERAL_REGISTERS(V) \ + ALWAYS_ALLOCATABLE_GENERAL_REGISTERS(V) \ + MAYBE_ALLOCATABLE_GENERAL_REGISTERS(V) + enum RegisterCode { #define REGISTER_CODE(R) kRegCode_##R, GENERAL_REGISTERS(REGISTER_CODE) @@ -146,7 +155,12 @@ constexpr Register arg_reg_4 = rcx; V(xmm13) \ V(xmm14) -constexpr bool kPadArguments = false; +// Returns the number of padding slots needed for stack pointer alignment. +constexpr int ArgumentPaddingSlots(int argument_count) { + // No argument padding required. + return 0; +} + constexpr bool kSimpleFPAliasing = true; constexpr bool kSimdMaskRegisters = false; @@ -201,7 +215,7 @@ constexpr Register kAllocateSizeRegister = rdx; constexpr Register kSpeculationPoisonRegister = r12; constexpr Register kInterpreterAccumulatorRegister = rax; constexpr Register kInterpreterBytecodeOffsetRegister = r9; -constexpr Register kInterpreterBytecodeArrayRegister = r14; +constexpr Register kInterpreterBytecodeArrayRegister = r12; constexpr Register kInterpreterDispatchTableRegister = r15; constexpr Register kJavaScriptCallArgCountRegister = rax; @@ -221,6 +235,11 @@ constexpr Register kWasmInstanceRegister = rsi; constexpr Register kScratchRegister = r10; constexpr XMMRegister kScratchDoubleReg = xmm15; constexpr Register kRootRegister = r13; // callee save +#ifdef V8_COMPRESS_POINTERS_IN_SHARED_CAGE +constexpr Register kPointerCageBaseRegister = r14; // callee save +#else +constexpr Register kPointerCageBaseRegister = kRootRegister; +#endif constexpr Register kOffHeapTrampolineRegister = kScratchRegister; |