diff options
Diffstat (limited to 'deps/v8/src/codegen/x64/assembler-x64.h')
| -rw-r--r-- | deps/v8/src/codegen/x64/assembler-x64.h | 211 |
1 files changed, 154 insertions, 57 deletions
diff --git a/deps/v8/src/codegen/x64/assembler-x64.h b/deps/v8/src/codegen/x64/assembler-x64.h index 8f77e38c1a..a8b7d77536 100644 --- a/deps/v8/src/codegen/x64/assembler-x64.h +++ b/deps/v8/src/codegen/x64/assembler-x64.h @@ -63,7 +63,7 @@ class MaglevSafepointTableBuilder; // Utility functions -enum Condition { +enum Condition : uint8_t { overflow = 0, no_overflow = 1, below = 2, @@ -88,6 +88,22 @@ enum Condition { not_zero = not_equal, sign = negative, not_sign = positive, + + // Unified cross-platform condition names/aliases. + kEqual = equal, + kNotEqual = not_equal, + kLessThan = less, + kGreaterThan = greater, + kLessThanEqual = less_equal, + kGreaterThanEqual = greater_equal, + kUnsignedLessThan = below, + kUnsignedGreaterThan = above, + kUnsignedLessThanEqual = below_equal, + kUnsignedGreaterThanEqual = above_equal, + kOverflow = overflow, + kNoOverflow = no_overflow, + kZero = equal, + kNotZero = not_equal, }; // Returns the equivalent of !cc. @@ -160,13 +176,46 @@ enum ScaleFactor : int8_t { class V8_EXPORT_PRIVATE Operand { public: - struct Data { - byte rex = 0; - byte buf[9] = {0}; - byte len = 1; // number of bytes of buf_ in use. - int8_t addend = 0; // for rip + offset + addend. + struct LabelOperand { + // The first two fields are shared in {LabelOperand} and {MemoryOperand}, + // but cannot be pulled out of the union, because otherwise the compiler + // introduces additional padding between them and the union, increasing the + // size unnecessarily. + bool is_label_operand = true; + byte rex = 0; // REX prefix, always zero for label operands. + + int8_t addend; // Used for rip + offset + addend operands. + Label* label; + }; + + struct MemoryOperand { + bool is_label_operand = false; + byte rex = 0; // REX prefix. + + // Register (1 byte) + SIB (0 or 1 byte) + displacement (0, 1, or 4 byte). + byte buf[6] = {0}; + // Number of bytes of buf in use. + // We must keep {len} and {buf} together for the compiler to elide the + // stack canary protection code. + size_t len = 1; }; + // Assert that the shared {is_label_operand} and {rex} fields have the same + // type and offset in both union variants. + static_assert(std::is_same<decltype(LabelOperand::is_label_operand), + decltype(MemoryOperand::is_label_operand)>::value); + static_assert(offsetof(LabelOperand, is_label_operand) == + offsetof(MemoryOperand, is_label_operand)); + static_assert(std::is_same<decltype(LabelOperand::rex), + decltype(MemoryOperand::rex)>::value); + static_assert(offsetof(LabelOperand, rex) == offsetof(MemoryOperand, rex)); + + static_assert(sizeof(MemoryOperand::len) == kSystemPointerSize, + "Length must have native word size to avoid spurious reloads " + "after writing it."); + static_assert(offsetof(MemoryOperand, len) % kSystemPointerSize == 0, + "Length must be aligned for fast access."); + // [base + disp/r] V8_INLINE constexpr Operand(Register base, int32_t disp) { if (base == rsp || base == r12) { @@ -218,17 +267,39 @@ class V8_EXPORT_PRIVATE Operand { // [rip + disp/r] V8_INLINE explicit Operand(Label* label, int addend = 0) { - data_.addend = addend; DCHECK_NOT_NULL(label); DCHECK(addend == 0 || (is_int8(addend) && label->is_bound())); - set_modrm(0, rbp); - set_disp64(reinterpret_cast<intptr_t>(label)); + label_ = {}; + label_.label = label; + label_.addend = addend; } Operand(const Operand&) V8_NOEXCEPT = default; Operand& operator=(const Operand&) V8_NOEXCEPT = default; - const Data& data() const { return data_; } + V8_INLINE constexpr bool is_label_operand() const { + // Since this field is in the common initial sequence of {label_} and + // {memory_}, the access is valid regardless of the active union member. + return memory_.is_label_operand; + } + + V8_INLINE constexpr byte rex() const { + // Since both fields are in the common initial sequence of {label_} and + // {memory_}, the access is valid regardless of the active union member. + // Label operands always have a REX prefix of zero. + V8_ASSUME(!memory_.is_label_operand || memory_.rex == 0); + return memory_.rex; + } + + V8_INLINE const MemoryOperand& memory() const { + DCHECK(!is_label_operand()); + return memory_; + } + + V8_INLINE const LabelOperand& label() const { + DCHECK(is_label_operand()); + return label_; + } // Checks whether either base or index register is the given register. // Does not check the "reg" part of the Operand. @@ -236,46 +307,43 @@ class V8_EXPORT_PRIVATE Operand { private: V8_INLINE constexpr void set_modrm(int mod, Register rm_reg) { + DCHECK(!is_label_operand()); DCHECK(is_uint2(mod)); - data_.buf[0] = mod << 6 | rm_reg.low_bits(); + memory_.buf[0] = mod << 6 | rm_reg.low_bits(); // Set REX.B to the high bit of rm.code(). - data_.rex |= rm_reg.high_bit(); + memory_.rex |= rm_reg.high_bit(); } V8_INLINE constexpr void set_sib(ScaleFactor scale, Register index, Register base) { - DCHECK_EQ(data_.len, 1); + V8_ASSUME(memory_.len == 1); DCHECK(is_uint2(scale)); // Use SIB with no index register only for base rsp or r12. Otherwise we // would skip the SIB byte entirely. DCHECK(index != rsp || base == rsp || base == r12); - data_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits(); - data_.rex |= index.high_bit() << 1 | base.high_bit(); - data_.len = 2; + memory_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits(); + memory_.rex |= index.high_bit() << 1 | base.high_bit(); + memory_.len = 2; } V8_INLINE constexpr void set_disp8(int disp) { + V8_ASSUME(memory_.len == 1 || memory_.len == 2); DCHECK(is_int8(disp)); - DCHECK(data_.len == 1 || data_.len == 2); - data_.buf[data_.len] = disp; - data_.len += sizeof(int8_t); + memory_.buf[memory_.len] = disp; + memory_.len += sizeof(int8_t); } V8_INLINE void set_disp32(int disp) { - DCHECK(data_.len == 1 || data_.len == 2); - Address p = reinterpret_cast<Address>(&data_.buf[data_.len]); - WriteUnalignedValue(p, disp); - data_.len += sizeof(int32_t); - } - - V8_INLINE void set_disp64(int64_t disp) { - DCHECK_EQ(1, data_.len); - Address p = reinterpret_cast<Address>(&data_.buf[data_.len]); + V8_ASSUME(memory_.len == 1 || memory_.len == 2); + Address p = reinterpret_cast<Address>(&memory_.buf[memory_.len]); WriteUnalignedValue(p, disp); - data_.len += sizeof(disp); + memory_.len += sizeof(int32_t); } - Data data_; + union { + LabelOperand label_; + MemoryOperand memory_ = {}; + }; }; class V8_EXPORT_PRIVATE Operand256 : public Operand { @@ -379,8 +447,7 @@ class ConstPool { Assembler* assm_; // Values, pc offsets of entries. - using EntryMap = std::multimap<uint64_t, int>; - EntryMap entries_; + std::multimap<uint64_t, int> entries_; // Number of bytes taken up by the displacement of rip-relative addressing. static constexpr int kRipRelativeDispSize = 4; // 32-bit displacement. @@ -443,12 +510,12 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { // Read/Modify the code target in the relative branch/call instruction at pc. // On the x64 architecture, we use relative jumps with a 32-bit displacement - // to jump to other Code objects in the Code space in the heap. - // Jumps to C functions are done indirectly through a 64-bit register holding - // the absolute address of the target. - // These functions convert between absolute Addresses of Code objects and - // the relative displacements stored in the code. - // The isolate argument is unused (and may be nullptr) when skipping flushing. + // to jump to other InstructionStream objects in the InstructionStream space + // in the heap. Jumps to C functions are done indirectly through a 64-bit + // register holding the absolute address of the target. These functions + // convert between absolute Addresses of InstructionStream objects and the + // relative displacements stored in the code. The isolate argument is unused + // (and may be nullptr) when skipping flushing. static inline Address target_address_at(Address pc, Address constant_pool); static inline void set_target_address_at( Address pc, Address constant_pool, Address target, @@ -474,7 +541,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { Address pc, Address target, RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE); - inline Handle<CodeT> code_target_object_handle_at(Address pc); + inline Handle<Code> code_target_object_handle_at(Address pc); inline Handle<HeapObject> compressed_embedded_object_handle_at(Address pc); // Number of bytes taken up by the branch target in the code. @@ -501,7 +568,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { enum LeadingOpcode { k0F = 0x1, k0F38 = 0x2, k0F3A = 0x3 }; // --------------------------------------------------------------------------- - // Code generation + // InstructionStream generation // // Function names correspond one-to-one to x64 instruction mnemonics. // Unless specified otherwise, instructions operate on 64-bit operands. @@ -823,7 +890,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { void near_jmp(intptr_t disp, RelocInfo::Mode rmode); void near_j(Condition cc, intptr_t disp, RelocInfo::Mode rmode); - void call(Handle<CodeT> target, + void call(Handle<Code> target, RelocInfo::Mode rmode = RelocInfo::CODE_TARGET); // Call near absolute indirect, address in register @@ -834,7 +901,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { // Use a 32-bit signed displacement. // Unconditional jump to L void jmp(Label* L, Label::Distance distance = Label::kFar); - void jmp(Handle<CodeT> target, RelocInfo::Mode rmode); + void jmp(Handle<Code> target, RelocInfo::Mode rmode); // Jump near absolute indirect (r64) void jmp(Register adr); @@ -847,7 +914,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { // Conditional jumps void j(Condition cc, Label* L, Label::Distance distance = Label::kFar); void j(Condition cc, Address entry, RelocInfo::Mode rmode); - void j(Condition cc, Handle<CodeT> target, RelocInfo::Mode rmode); + void j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode); // Floating-point operations void fld(int i); @@ -1524,9 +1591,21 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { void vcvtdq2pd(XMMRegister dst, XMMRegister src) { vinstr(0xe6, dst, xmm0, src, kF3, k0F, kWIG); } + void vcvtdq2pd(YMMRegister dst, XMMRegister src) { + vinstr(0xe6, dst, xmm0, src, kF3, k0F, kWIG, AVX); + } + void vcvtdq2pd(YMMRegister dst, Operand src) { + vinstr(0xe6, dst, xmm0, src, kF3, k0F, kWIG, AVX); + } void vcvttps2dq(XMMRegister dst, XMMRegister src) { vinstr(0x5b, dst, xmm0, src, kF3, k0F, kWIG); } + void vcvttps2dq(YMMRegister dst, YMMRegister src) { + vinstr(0x5b, dst, ymm0, src, kF3, k0F, kWIG, AVX); + } + void vcvttps2dq(YMMRegister dst, Operand src) { + vinstr(0x5b, dst, ymm0, src, kF3, k0F, kWIG, AVX); + } void vcvtlsi2sd(XMMRegister dst, XMMRegister src1, Register src2) { XMMRegister isrc2 = XMMRegister::from_code(src2.code()); vinstr(0x2a, dst, src1, isrc2, kF2, k0F, kW0); @@ -2088,12 +2167,12 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { // Check if there is less than kGap bytes available in the buffer. // If this is the case, we need to grow the buffer before emitting // an instruction or relocation information. - inline bool buffer_overflow() const { - return pc_ >= reloc_info_writer.pos() - kGap; - } + bool buffer_overflow() const { return available_space() < kGap; } // Get the number of bytes available in the buffer. - inline int available_space() const { + int available_space() const { + DCHECK_GE(reloc_info_writer.pos(), pc_); + DCHECK_GE(kMaxInt, reloc_info_writer.pos() - pc_); return static_cast<int>(reloc_info_writer.pos() - pc_); } @@ -2126,15 +2205,29 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { WriteUnalignedValue(addr_at(pos), x); } - // code emission - void GrowBuffer(); + // InstructionStream emission. + V8_NOINLINE V8_PRESERVE_MOST void GrowBuffer(); - void emit(byte x) { *pc_++ = x; } - inline void emitl(uint32_t x); - inline void emitq(uint64_t x); - inline void emitw(uint16_t x); - inline void emit(Immediate x); - inline void emit(Immediate64 x); + template <typename T> + static uint8_t* emit(uint8_t* __restrict pc, T t) { + WriteUnalignedValue(reinterpret_cast<Address>(pc), t); + return pc + sizeof(T); + } + + void emit(uint8_t x) { pc_ = emit(pc_, x); } + void emitw(uint16_t x) { pc_ = emit(pc_, x); } + void emitl(uint32_t x) { pc_ = emit(pc_, x); } + void emitq(uint64_t x) { pc_ = emit(pc_, x); } + + void emit(Immediate x) { + if (!RelocInfo::IsNoInfo(x.rmode_)) RecordRelocInfo(x.rmode_); + emitl(x.value_); + } + + void emit(Immediate64 x) { + if (!RelocInfo::IsNoInfo(x.rmode_)) RecordRelocInfo(x.rmode_); + emitq(static_cast<uint64_t>(x.value_)); + } // Emits a REX prefix that encodes a 64-bit operand size and // the top bit of both register codes. @@ -2287,10 +2380,14 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase { } // Emit the ModR/M byte, and optionally the SIB byte and - // 1- or 4-byte offset for a memory operand. Also used to encode - // a three-bit opcode extension into the ModR/M byte. + // 1- or 4-byte offset for a memory operand. + // Also used to encode a three-bit opcode extension into the ModR/M byte. void emit_operand(int rm, Operand adr); + // Emit a RIP-relative operand. + // Also used to encode a three-bit opcode extension into the ModR/M byte. + V8_NOINLINE void emit_label_operand(int rm, Label* label, int addend = 0); + // Emit a ModR/M byte with registers coded in the reg and rm_reg fields. void emit_modrm(Register reg, Register rm_reg) { emit(0xC0 | reg.low_bits() << 3 | rm_reg.low_bits()); |