diff options
Diffstat (limited to 'deps/v8/src/codegen/loong64/assembler-loong64.cc')
| -rw-r--r-- | deps/v8/src/codegen/loong64/assembler-loong64.cc | 2405 |
1 files changed, 2405 insertions, 0 deletions
diff --git a/deps/v8/src/codegen/loong64/assembler-loong64.cc b/deps/v8/src/codegen/loong64/assembler-loong64.cc new file mode 100644 index 0000000000..cc1eaa7d12 --- /dev/null +++ b/deps/v8/src/codegen/loong64/assembler-loong64.cc @@ -0,0 +1,2405 @@ +// Copyright 2021 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/codegen/loong64/assembler-loong64.h" + +#if V8_TARGET_ARCH_LOONG64 + +#include "src/base/cpu.h" +#include "src/codegen/loong64/assembler-loong64-inl.h" +#include "src/codegen/machine-type.h" +#include "src/codegen/safepoint-table.h" +#include "src/codegen/string-constants.h" +#include "src/deoptimizer/deoptimizer.h" +#include "src/objects/heap-number-inl.h" + +namespace v8 { +namespace internal { + +bool CpuFeatures::SupportsWasmSimd128() { return false; } + +void CpuFeatures::ProbeImpl(bool cross_compile) { + supported_ |= 1u << FPU; + + // Only use statically determined features for cross compile (snapshot). + if (cross_compile) return; + +#ifdef __loongarch__ + // Probe for additional features at runtime. + base::CPU cpu; + supported_ |= 1u << FPU; +#endif + + // Set a static value on whether Simd is supported. + // This variable is only used for certain archs to query SupportWasmSimd128() + // at runtime in builtins using an extern ref. Other callers should use + // CpuFeatures::SupportWasmSimd128(). + CpuFeatures::supports_wasm_simd_128_ = CpuFeatures::SupportsWasmSimd128(); +} + +void CpuFeatures::PrintTarget() {} +void CpuFeatures::PrintFeatures() {} + +int ToNumber(Register reg) { + DCHECK(reg.is_valid()); + const int kNumbers[] = { + 0, // zero_reg + 1, // ra + 2, // tp + 3, // sp + 4, // a0 v0 + 5, // a1 v1 + 6, // a2 + 7, // a3 + 8, // a4 + 9, // a5 + 10, // a6 + 11, // a7 + 12, // t0 + 13, // t1 + 14, // t2 + 15, // t3 + 16, // t4 + 17, // t5 + 18, // t6 + 19, // t7 + 20, // t8 + 21, // x_reg + 22, // fp + 23, // s0 + 24, // s1 + 25, // s2 + 26, // s3 + 27, // s4 + 28, // s5 + 29, // s6 + 30, // s7 + 31, // s8 + }; + return kNumbers[reg.code()]; +} + +Register ToRegister(int num) { + DCHECK(num >= 0 && num < kNumRegisters); + const Register kRegisters[] = { + zero_reg, ra, tp, sp, a0, a1, a2, a3, a4, a5, a6, a7, t0, t1, t2, t3, + t4, t5, t6, t7, t8, x_reg, fp, s0, s1, s2, s3, s4, s5, s6, s7, s8}; + return kRegisters[num]; +} + +// ----------------------------------------------------------------------------- +// Implementation of RelocInfo. + +const int RelocInfo::kApplyMask = + RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) | + RelocInfo::ModeMask(RelocInfo::RELATIVE_CODE_TARGET); + +bool RelocInfo::IsCodedSpecially() { + // The deserializer needs to know whether a pointer is specially coded. Being + // specially coded on LoongArch64 means that it is a lu12i_w/ori instruction, + // and that is always the case inside code objects. + return true; +} + +bool RelocInfo::IsInConstantPool() { return false; } + +uint32_t RelocInfo::wasm_call_tag() const { + DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL); + return static_cast<uint32_t>( + Assembler::target_address_at(pc_, constant_pool_)); +} + +// ----------------------------------------------------------------------------- +// Implementation of Operand and MemOperand. +// See assembler-loong64-inl.h for inlined constructors. + +Operand::Operand(Handle<HeapObject> handle) + : rm_(no_reg), rmode_(RelocInfo::FULL_EMBEDDED_OBJECT) { + value_.immediate = static_cast<intptr_t>(handle.address()); +} + +Operand Operand::EmbeddedNumber(double value) { + int32_t smi; + if (DoubleToSmiInteger(value, &smi)) return Operand(Smi::FromInt(smi)); + Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT); + result.is_heap_object_request_ = true; + result.value_.heap_object_request = HeapObjectRequest(value); + return result; +} + +Operand Operand::EmbeddedStringConstant(const StringConstantBase* str) { + Operand result(0, RelocInfo::FULL_EMBEDDED_OBJECT); + result.is_heap_object_request_ = true; + result.value_.heap_object_request = HeapObjectRequest(str); + return result; +} + +MemOperand::MemOperand(Register base, int32_t offset) + : base_(base), index_(no_reg), offset_(offset) {} + +MemOperand::MemOperand(Register base, Register index) + : base_(base), index_(index), offset_(0) {} + +void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) { + DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty()); + for (auto& request : heap_object_requests_) { + Handle<HeapObject> object; + switch (request.kind()) { + case HeapObjectRequest::kHeapNumber: + object = isolate->factory()->NewHeapNumber<AllocationType::kOld>( + request.heap_number()); + break; + case HeapObjectRequest::kStringConstant: + const StringConstantBase* str = request.string(); + CHECK_NOT_NULL(str); + object = str->AllocateStringConstant(isolate); + break; + } + Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset(); + set_target_value_at(pc, reinterpret_cast<uint64_t>(object.location())); + } +} + +// ----------------------------------------------------------------------------- +// Specific instructions, constants, and masks. + +Assembler::Assembler(const AssemblerOptions& options, + std::unique_ptr<AssemblerBuffer> buffer) + : AssemblerBase(options, std::move(buffer)), + scratch_register_list_(t7.bit() | t6.bit()) { + reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_); + + last_trampoline_pool_end_ = 0; + no_trampoline_pool_before_ = 0; + trampoline_pool_blocked_nesting_ = 0; + // We leave space (16 * kTrampolineSlotsSize) + // for BlockTrampolinePoolScope buffer. + next_buffer_check_ = FLAG_force_long_branches + ? kMaxInt + : kMax16BranchOffset - kTrampolineSlotsSize * 16; + internal_trampoline_exception_ = false; + last_bound_pos_ = 0; + + trampoline_emitted_ = FLAG_force_long_branches; + unbound_labels_count_ = 0; + block_buffer_growth_ = false; +} + +void Assembler::GetCode(Isolate* isolate, CodeDesc* desc, + SafepointTableBuilder* safepoint_table_builder, + int handler_table_offset) { + // As a crutch to avoid having to add manual Align calls wherever we use a + // raw workflow to create Code objects (mostly in tests), add another Align + // call here. It does no harm - the end of the Code object is aligned to the + // (larger) kCodeAlignment anyways. + // TODO(jgruber): Consider moving responsibility for proper alignment to + // metadata table builders (safepoint, handler, constant pool, code + // comments). + DataAlign(Code::kMetadataAlignment); + + // EmitForbiddenSlotInstruction(); TODO:LOONG64 why? + + int code_comments_size = WriteCodeComments(); + + DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap. + + AllocateAndInstallRequestedHeapObjects(isolate); + + // Set up code descriptor. + // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to + // this point to make CodeDesc initialization less fiddly. + + static constexpr int kConstantPoolSize = 0; + const int instruction_size = pc_offset(); + const int code_comments_offset = instruction_size - code_comments_size; + const int constant_pool_offset = code_comments_offset - kConstantPoolSize; + const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable) + ? constant_pool_offset + : handler_table_offset; + const int safepoint_table_offset = + (safepoint_table_builder == kNoSafepointTable) + ? handler_table_offset2 + : safepoint_table_builder->GetCodeOffset(); + const int reloc_info_offset = + static_cast<int>(reloc_info_writer.pos() - buffer_->start()); + CodeDesc::Initialize(desc, this, safepoint_table_offset, + handler_table_offset2, constant_pool_offset, + code_comments_offset, reloc_info_offset); +} + +void Assembler::Align(int m) { + // If not, the loop below won't terminate. + DCHECK(IsAligned(pc_offset(), kInstrSize)); + DCHECK(m >= kInstrSize && base::bits::IsPowerOfTwo(m)); + while ((pc_offset() & (m - 1)) != 0) { + nop(); + } +} + +void Assembler::CodeTargetAlign() { + // No advantage to aligning branch/call targets to more than + // single instruction, that I am aware of. + Align(4); +} + +Register Assembler::GetRkReg(Instr instr) { + return Register::from_code((instr & kRkFieldMask) >> kRkShift); +} + +Register Assembler::GetRjReg(Instr instr) { + return Register::from_code((instr & kRjFieldMask) >> kRjShift); +} + +Register Assembler::GetRdReg(Instr instr) { + return Register::from_code((instr & kRdFieldMask) >> kRdShift); +} + +uint32_t Assembler::GetRk(Instr instr) { + return (instr & kRkFieldMask) >> kRkShift; +} + +uint32_t Assembler::GetRkField(Instr instr) { return instr & kRkFieldMask; } + +uint32_t Assembler::GetRj(Instr instr) { + return (instr & kRjFieldMask) >> kRjShift; +} + +uint32_t Assembler::GetRjField(Instr instr) { return instr & kRjFieldMask; } + +uint32_t Assembler::GetRd(Instr instr) { + return (instr & kRdFieldMask) >> kRdShift; +} + +uint32_t Assembler::GetRdField(Instr instr) { return instr & kRdFieldMask; } + +uint32_t Assembler::GetSa2(Instr instr) { + return (instr & kSa2FieldMask) >> kSaShift; +} + +uint32_t Assembler::GetSa2Field(Instr instr) { return instr & kSa2FieldMask; } + +uint32_t Assembler::GetSa3(Instr instr) { + return (instr & kSa3FieldMask) >> kSaShift; +} + +uint32_t Assembler::GetSa3Field(Instr instr) { return instr & kSa3FieldMask; } + +// Labels refer to positions in the (to be) generated code. +// There are bound, linked, and unused labels. +// +// Bound labels refer to known positions in the already +// generated code. pos() is the position the label refers to. +// +// Linked labels refer to unknown positions in the code +// to be generated; pos() is the position of the last +// instruction using the label. + +// The link chain is terminated by a value in the instruction of 0, +// which is an otherwise illegal value (branch 0 is inf loop). +// The instruction 16-bit offset field addresses 32-bit words, but in +// code is conv to an 18-bit value addressing bytes, hence the -4 value. + +const int kEndOfChain = 0; +// Determines the end of the Jump chain (a subset of the label link chain). +const int kEndOfJumpChain = 0; + +bool Assembler::IsBranch(Instr instr) { + uint32_t opcode = (instr >> 26) << 26; + // Checks if the instruction is a branch. + bool isBranch = opcode == BEQZ || opcode == BNEZ || opcode == BCZ || + opcode == B || opcode == BL || opcode == BEQ || + opcode == BNE || opcode == BLT || opcode == BGE || + opcode == BLTU || opcode == BGEU; + return isBranch; +} + +bool Assembler::IsB(Instr instr) { + uint32_t opcode = (instr >> 26) << 26; + // Checks if the instruction is a b. + bool isBranch = opcode == B || opcode == BL; + return isBranch; +} + +bool Assembler::IsBz(Instr instr) { + uint32_t opcode = (instr >> 26) << 26; + // Checks if the instruction is a branch. + bool isBranch = opcode == BEQZ || opcode == BNEZ || opcode == BCZ; + return isBranch; +} + +bool Assembler::IsEmittedConstant(Instr instr) { + // Add GetLabelConst function? + uint32_t label_constant = instr & ~kImm16Mask; + return label_constant == 0; // Emitted label const in reg-exp engine. +} + +bool Assembler::IsJ(Instr instr) { + uint32_t opcode = (instr >> 26) << 26; + // Checks if the instruction is a jump. + return opcode == JIRL; +} + +bool Assembler::IsLu12i_w(Instr instr) { + uint32_t opcode = (instr >> 25) << 25; + return opcode == LU12I_W; +} + +bool Assembler::IsOri(Instr instr) { + uint32_t opcode = (instr >> 22) << 22; + return opcode == ORI; +} + +bool Assembler::IsLu32i_d(Instr instr) { + uint32_t opcode = (instr >> 25) << 25; + return opcode == LU32I_D; +} + +bool Assembler::IsLu52i_d(Instr instr) { + uint32_t opcode = (instr >> 22) << 22; + return opcode == LU52I_D; +} + +bool Assembler::IsMov(Instr instr, Register rd, Register rj) { + // Checks if the instruction is a OR with zero_reg argument (aka MOV). + Instr instr1 = + OR | zero_reg.code() << kRkShift | rj.code() << kRjShift | rd.code(); + return instr == instr1; +} + +bool Assembler::IsPcAddi(Instr instr, Register rd, int32_t si20) { + DCHECK(is_int20(si20)); + Instr instr1 = PCADDI | (si20 & 0xfffff) << kRjShift | rd.code(); + return instr == instr1; +} + +bool Assembler::IsNop(Instr instr, unsigned int type) { + // See Assembler::nop(type). + DCHECK_LT(type, 32); + + Instr instr1 = + ANDI | ((type & kImm12Mask) << kRkShift) | (zero_reg.code() << kRjShift); + + return instr == instr1; +} + +static inline int32_t GetOffsetOfBranch(Instr instr, + Assembler::OffsetSize bits) { + int32_t result = 0; + if (bits == 16) { + result = (instr << 6) >> 16; + } else if (bits == 21) { + uint32_t low16 = instr << 6; + low16 = low16 >> 16; + low16 &= 0xffff; + int32_t hi5 = (instr << 27) >> 11; + result = hi5 | low16; + } else { + uint32_t low16 = instr << 6; + low16 = low16 >> 16; + low16 &= 0xffff; + int32_t hi10 = (instr << 22) >> 6; + result = hi10 | low16; + DCHECK_EQ(bits, 26); + } + return result << 2; +} + +static Assembler::OffsetSize OffsetSizeInBits(Instr instr) { + if (Assembler::IsB(instr)) { + return Assembler::OffsetSize::kOffset26; + } else if (Assembler::IsBz(instr)) { + return Assembler::OffsetSize::kOffset21; + } else { + DCHECK(Assembler::IsBranch(instr)); + return Assembler::OffsetSize::kOffset16; + } +} + +static inline int32_t AddBranchOffset(int pos, Instr instr) { + Assembler::OffsetSize bits = OffsetSizeInBits(instr); + + int32_t imm = GetOffsetOfBranch(instr, bits); + + if (imm == kEndOfChain) { + // EndOfChain sentinel is returned directly, not relative to pc or pos. + return kEndOfChain; + } else { + // Handle the case that next branch position is 0. + // TODO(LOONG_dev): Define -4 as a constant + int32_t offset = pos + imm; + return offset == 0 ? -4 : offset; + } +} + +int Assembler::target_at(int pos, bool is_internal) { + if (is_internal) { + int64_t* p = reinterpret_cast<int64_t*>(buffer_start_ + pos); + int64_t address = *p; + if (address == kEndOfJumpChain) { + return kEndOfChain; + } else { + int64_t instr_address = reinterpret_cast<int64_t>(p); + DCHECK(instr_address - address < INT_MAX); + int delta = static_cast<int>(instr_address - address); + DCHECK(pos > delta); + return pos - delta; + } + } + Instr instr = instr_at(pos); + + // TODO(LOONG_dev) remove after remove label_at_put? + if ((instr & ~kImm16Mask) == 0) { + // Emitted label constant, not part of a branch. + if (instr == 0) { + return kEndOfChain; + } else { + int32_t imm18 = ((instr & static_cast<int32_t>(kImm16Mask)) << 16) >> 14; + return (imm18 + pos); + } + } + + // Check we have a branch or jump instruction. + DCHECK(IsBranch(instr) || IsPcAddi(instr, t8, 16)); + // Do NOT change this to <<2. We rely on arithmetic shifts here, assuming + // the compiler uses arithmetic shifts for signed integers. + if (IsBranch(instr)) { + return AddBranchOffset(pos, instr); + } else { + DCHECK(IsPcAddi(instr, t8, 16)); + // see BranchLong(Label* L) and BranchAndLinkLong ?? + int32_t imm32; + Instr instr_lu12i_w = instr_at(pos + 1 * kInstrSize); + Instr instr_ori = instr_at(pos + 2 * kInstrSize); + DCHECK(IsLu12i_w(instr_lu12i_w)); + imm32 = ((instr_lu12i_w >> 5) & 0xfffff) << 12; + imm32 |= ((instr_ori >> 10) & static_cast<int32_t>(kImm12Mask)); + if (imm32 == kEndOfJumpChain) { + // EndOfChain sentinel is returned directly, not relative to pc or pos. + return kEndOfChain; + } + return pos + imm32; + } +} + +static inline Instr SetBranchOffset(int32_t pos, int32_t target_pos, + Instr instr) { + int32_t bits = OffsetSizeInBits(instr); + int32_t imm = target_pos - pos; + DCHECK_EQ(imm & 3, 0); + imm >>= 2; + + DCHECK(is_intn(imm, bits)); + + if (bits == 16) { + const int32_t mask = ((1 << 16) - 1) << 10; + instr &= ~mask; + return instr | ((imm << 10) & mask); + } else if (bits == 21) { + const int32_t mask = 0x3fffc1f; + instr &= ~mask; + uint32_t low16 = (imm & kImm16Mask) << 10; + int32_t hi5 = (imm >> 16) & 0x1f; + return instr | low16 | hi5; + } else { + DCHECK_EQ(bits, 26); + const int32_t mask = 0x3ffffff; + instr &= ~mask; + uint32_t low16 = (imm & kImm16Mask) << 10; + int32_t hi10 = (imm >> 16) & 0x3ff; + return instr | low16 | hi10; + } +} + +void Assembler::target_at_put(int pos, int target_pos, bool is_internal) { + if (is_internal) { + uint64_t imm = reinterpret_cast<uint64_t>(buffer_start_) + target_pos; + *reinterpret_cast<uint64_t*>(buffer_start_ + pos) = imm; + return; + } + Instr instr = instr_at(pos); + if ((instr & ~kImm16Mask) == 0) { + DCHECK(target_pos == kEndOfChain || target_pos >= 0); + // Emitted label constant, not part of a branch. + // Make label relative to Code pointer of generated Code object. + instr_at_put(pos, target_pos + (Code::kHeaderSize - kHeapObjectTag)); + return; + } + + DCHECK(IsBranch(instr)); + instr = SetBranchOffset(pos, target_pos, instr); + instr_at_put(pos, instr); +} + +void Assembler::print(const Label* L) { + if (L->is_unused()) { + PrintF("unused label\n"); + } else if (L->is_bound()) { + PrintF("bound label to %d\n", L->pos()); + } else if (L->is_linked()) { + Label l; + l.link_to(L->pos()); + PrintF("unbound label"); + while (l.is_linked()) { + PrintF("@ %d ", l.pos()); + Instr instr = instr_at(l.pos()); + if ((instr & ~kImm16Mask) == 0) { + PrintF("value\n"); + } else { + PrintF("%d\n", instr); + } + next(&l, is_internal_reference(&l)); + } + } else { + PrintF("label in inconsistent state (pos = %d)\n", L->pos_); + } +} + +void Assembler::bind_to(Label* L, int pos) { + DCHECK(0 <= pos && pos <= pc_offset()); // Must have valid binding position. + int trampoline_pos = kInvalidSlotPos; + bool is_internal = false; + if (L->is_linked() && !trampoline_emitted_) { + unbound_labels_count_--; + if (!is_internal_reference(L)) { + next_buffer_check_ += kTrampolineSlotsSize; + } + } + + while (L->is_linked()) { + int fixup_pos = L->pos(); + int dist = pos - fixup_pos; + is_internal = is_internal_reference(L); + next(L, is_internal); // Call next before overwriting link with target at + // fixup_pos. + Instr instr = instr_at(fixup_pos); + if (is_internal) { + target_at_put(fixup_pos, pos, is_internal); + } else { + if (IsBranch(instr)) { + int branch_offset = BranchOffset(instr); + if (dist > branch_offset) { + if (trampoline_pos == kInvalidSlotPos) { + trampoline_pos = get_trampoline_entry(fixup_pos); + CHECK_NE(trampoline_pos, kInvalidSlotPos); + } + CHECK((trampoline_pos - fixup_pos) <= branch_offset); + target_at_put(fixup_pos, trampoline_pos, false); + fixup_pos = trampoline_pos; + } + target_at_put(fixup_pos, pos, false); + } else { + DCHECK(IsJ(instr) || IsLu12i_w(instr) || IsEmittedConstant(instr) || + IsPcAddi(instr, t8, 8)); + target_at_put(fixup_pos, pos, false); + } + } + } + L->bind_to(pos); + + // Keep track of the last bound label so we don't eliminate any instructions + // before a bound label. + if (pos > last_bound_pos_) last_bound_pos_ = pos; +} + +void Assembler::bind(Label* L) { + DCHECK(!L->is_bound()); // Label can only be bound once. + bind_to(L, pc_offset()); +} + +void Assembler::next(Label* L, bool is_internal) { + DCHECK(L->is_linked()); + int link = target_at(L->pos(), is_internal); + if (link == kEndOfChain) { + L->Unuse(); + } else if (link == -4) { + // Next position is pc_offset == 0 + L->link_to(0); + } else { + DCHECK_GE(link, 0); + L->link_to(link); + } +} + +bool Assembler::is_near_c(Label* L) { + DCHECK(L->is_bound()); + return pc_offset() - L->pos() < kMax16BranchOffset - 4 * kInstrSize; +} + +bool Assembler::is_near(Label* L, OffsetSize bits) { + DCHECK(L->is_bound()); + return ((pc_offset() - L->pos()) < + (1 << (bits + 2 - 1)) - 1 - 5 * kInstrSize); +} + +bool Assembler::is_near_a(Label* L) { + DCHECK(L->is_bound()); + return pc_offset() - L->pos() <= kMax26BranchOffset - 4 * kInstrSize; +} + +int Assembler::BranchOffset(Instr instr) { + int bits = OffsetSize::kOffset16; + + uint32_t opcode = (instr >> 26) << 26; + switch (opcode) { + case B: + case BL: + bits = OffsetSize::kOffset26; + break; + case BNEZ: + case BEQZ: + case BCZ: + bits = OffsetSize::kOffset21; + break; + case BNE: + case BEQ: + case BLT: + case BGE: + case BLTU: + case BGEU: + case JIRL: + bits = OffsetSize::kOffset16; + break; + default: + break; + } + + return (1 << (bits + 2 - 1)) - 1; +} + +// We have to use a temporary register for things that can be relocated even +// if they can be encoded in the LOONG's 16 bits of immediate-offset +// instruction space. There is no guarantee that the relocated location can be +// similarly encoded. +bool Assembler::MustUseReg(RelocInfo::Mode rmode) { + return !RelocInfo::IsNone(rmode); +} + +void Assembler::GenB(Opcode opcode, Register rj, int32_t si21) { + BlockTrampolinePoolScope block_trampoline_pool(this); + DCHECK((BEQZ == opcode || BNEZ == opcode) && is_int21(si21) && rj.is_valid()); + Instr instr = opcode | (si21 & kImm16Mask) << kRkShift | + (rj.code() << kRjShift) | ((si21 & 0x1fffff) >> 16); + emit(instr); +} + +void Assembler::GenB(Opcode opcode, CFRegister cj, int32_t si21, bool isEq) { + BlockTrampolinePoolScope block_trampoline_pool(this); + DCHECK(BCZ == opcode && is_int21(si21)); + DCHECK(cj >= 0 && cj <= 7); + int32_t sc = (isEq ? cj : cj + 8); + Instr instr = opcode | (si21 & kImm16Mask) << kRkShift | (sc << kRjShift) | + ((si21 & 0x1fffff) >> 16); + emit(instr); +} + +void Assembler::GenB(Opcode opcode, int32_t si26) { + BlockTrampolinePoolScope block_trampoline_pool(this); + DCHECK((B == opcode || BL == opcode) && is_int26(si26)); + Instr instr = + opcode | ((si26 & kImm16Mask) << kRkShift) | ((si26 & kImm26Mask) >> 16); + emit(instr); +} + +void Assembler::GenBJ(Opcode opcode, Register rj, Register rd, int32_t si16) { + BlockTrampolinePoolScope block_trampoline_pool(this); + DCHECK(is_int16(si16)); + Instr instr = opcode | ((si16 & kImm16Mask) << kRkShift) | + (rj.code() << kRjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenCmp(Opcode opcode, FPUCondition cond, FPURegister fk, + FPURegister fj, CFRegister cd) { + DCHECK(opcode == FCMP_COND_S || opcode == FCMP_COND_D); + Instr instr = opcode | cond << kCondShift | (fk.code() << kFkShift) | + (fj.code() << kFjShift) | cd; + emit(instr); +} + +void Assembler::GenSel(Opcode opcode, CFRegister ca, FPURegister fk, + FPURegister fj, FPURegister rd) { + DCHECK((opcode == FSEL)); + Instr instr = opcode | ca << kCondShift | (fk.code() << kFkShift) | + (fj.code() << kFjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, Register rj, Register rd, + bool rjrd) { + DCHECK(rjrd); + Instr instr = 0; + instr = opcode | (rj.code() << kRjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, FPURegister fj, FPURegister fd) { + Instr instr = opcode | (fj.code() << kFjShift) | fd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, Register rj, FPURegister fd) { + DCHECK((opcode == MOVGR2FR_W) || (opcode == MOVGR2FR_D) || + (opcode == MOVGR2FRH_W)); + Instr instr = opcode | (rj.code() << kRjShift) | fd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, FPURegister fj, Register rd) { + DCHECK((opcode == MOVFR2GR_S) || (opcode == MOVFR2GR_D) || + (opcode == MOVFRH2GR_S)); + Instr instr = opcode | (fj.code() << kFjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, Register rj, FPUControlRegister fd) { + DCHECK((opcode == MOVGR2FCSR)); + Instr instr = opcode | (rj.code() << kRjShift) | fd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, FPUControlRegister fj, Register rd) { + DCHECK((opcode == MOVFCSR2GR)); + Instr instr = opcode | (fj.code() << kFjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, FPURegister fj, CFRegister cd) { + DCHECK((opcode == MOVFR2CF)); + Instr instr = opcode | (fj.code() << kFjShift) | cd; + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, CFRegister cj, FPURegister fd) { + DCHECK((opcode == MOVCF2FR)); + Instr instr = opcode | cj << kFjShift | fd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, Register rj, CFRegister cd) { + DCHECK((opcode == MOVGR2CF)); + Instr instr = opcode | (rj.code() << kRjShift) | cd; + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, CFRegister cj, Register rd) { + DCHECK((opcode == MOVCF2GR)); + Instr instr = opcode | cj << kFjShift | rd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, Register rk, Register rj, + Register rd) { + Instr instr = + opcode | (rk.code() << kRkShift) | (rj.code() << kRjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, FPURegister fk, FPURegister fj, + FPURegister fd) { + Instr instr = + opcode | (fk.code() << kFkShift) | (fj.code() << kFjShift) | fd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, FPURegister fa, FPURegister fk, + FPURegister fj, FPURegister fd) { + Instr instr = opcode | (fa.code() << kFaShift) | (fk.code() << kFkShift) | + (fj.code() << kFjShift) | fd.code(); + emit(instr); +} + +void Assembler::GenRegister(Opcode opcode, Register rk, Register rj, + FPURegister fd) { + Instr instr = + opcode | (rk.code() << kRkShift) | (rj.code() << kRjShift) | fd.code(); + emit(instr); +} + +void Assembler::GenImm(Opcode opcode, int32_t bit3, Register rk, Register rj, + Register rd) { + DCHECK(is_uint3(bit3)); + Instr instr = opcode | (bit3 & 0x7) << kSaShift | (rk.code() << kRkShift) | + (rj.code() << kRjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenImm(Opcode opcode, int32_t bit6m, int32_t bit6l, Register rj, + Register rd) { + DCHECK(is_uint6(bit6m) && is_uint6(bit6l)); + Instr instr = opcode | (bit6m & 0x3f) << 16 | (bit6l & 0x3f) << kRkShift | + (rj.code() << kRjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenImm(Opcode opcode, int32_t bit20, Register rd) { + // DCHECK(is_uint20(bit20) || is_int20(bit20)); + Instr instr = opcode | (bit20 & 0xfffff) << kRjShift | rd.code(); + emit(instr); +} + +void Assembler::GenImm(Opcode opcode, int32_t bit15) { + DCHECK(is_uint15(bit15)); + Instr instr = opcode | (bit15 & 0x7fff); + emit(instr); +} + +void Assembler::GenImm(Opcode opcode, int32_t value, Register rj, Register rd, + int32_t value_bits) { + DCHECK(value_bits == 6 || value_bits == 12 || value_bits == 14 || + value_bits == 16); + uint32_t imm = value & 0x3f; + if (value_bits == 12) { + imm = value & kImm12Mask; + } else if (value_bits == 14) { + imm = value & 0x3fff; + } else if (value_bits == 16) { + imm = value & kImm16Mask; + } + Instr instr = opcode | imm << kRkShift | (rj.code() << kRjShift) | rd.code(); + emit(instr); +} + +void Assembler::GenImm(Opcode opcode, int32_t bit12, Register rj, + FPURegister fd) { + DCHECK(is_int12(bit12)); + Instr instr = opcode | ((bit12 & kImm12Mask) << kRkShift) | + (rj.code() << kRjShift) | fd.code(); + emit(instr); +} + +// Returns the next free trampoline entry. +int32_t Assembler::get_trampoline_entry(int32_t pos) { + int32_t trampoline_entry = kInvalidSlotPos; + if (!internal_trampoline_exception_) { + if (trampoline_.start() > pos) { + trampoline_entry = trampoline_.take_slot(); + } + + if (kInvalidSlotPos == trampoline_entry) { + internal_trampoline_exception_ = true; + } + } + return trampoline_entry; +} + +uint64_t Assembler::jump_address(Label* L) { + int64_t target_pos; + if (L->is_bound()) { + target_pos = L->pos(); + } else { + if (L->is_linked()) { + target_pos = L->pos(); // L's link. + L->link_to(pc_offset()); + } else { + L->link_to(pc_offset()); + return kEndOfJumpChain; + } + } + uint64_t imm = reinterpret_cast<uint64_t>(buffer_start_) + target_pos; + DCHECK_EQ(imm & 3, 0); + + return imm; +} + +uint64_t Assembler::branch_long_offset(Label* L) { + int64_t target_pos; + + if (L->is_bound()) { + target_pos = L->pos(); + } else { + if (L->is_linked()) { + target_pos = L->pos(); // L's link. + L->link_to(pc_offset()); + } else { + L->link_to(pc_offset()); + return kEndOfJumpChain; + } + } + int64_t offset = target_pos - pc_offset(); + DCHECK_EQ(offset & 3, 0); + + return static_cast<uint64_t>(offset); +} + +int32_t Assembler::branch_offset_helper(Label* L, OffsetSize bits) { + int32_t target_pos; + + if (L->is_bound()) { + target_pos = L->pos(); + } else { + if (L->is_linked()) { + target_pos = L->pos(); + L->link_to(pc_offset()); + } else { + L->link_to(pc_offset()); + if (!trampoline_emitted_) { + unbound_labels_count_++; + next_buffer_check_ -= kTrampolineSlotsSize; + } + return kEndOfChain; + } + } + + int32_t offset = target_pos - pc_offset(); + DCHECK(is_intn(offset, bits + 2)); + DCHECK_EQ(offset & 3, 0); + + return offset; +} + +void Assembler::label_at_put(Label* L, int at_offset) { + int target_pos; + if (L->is_bound()) { + target_pos = L->pos(); + instr_at_put(at_offset, target_pos + (Code::kHeaderSize - kHeapObjectTag)); + } else { + if (L->is_linked()) { + target_pos = L->pos(); // L's link. + int32_t imm18 = target_pos - at_offset; + DCHECK_EQ(imm18 & 3, 0); + int32_t imm16 = imm18 >> 2; + DCHECK(is_int16(imm16)); + instr_at_put(at_offset, (imm16 & kImm16Mask)); + } else { + target_pos = kEndOfChain; + instr_at_put(at_offset, 0); + if (!trampoline_emitted_) { + unbound_labels_count_++; + next_buffer_check_ -= kTrampolineSlotsSize; + } + } + L->link_to(at_offset); + } +} + +//------- Branch and jump instructions -------- + +void Assembler::b(int32_t offset) { GenB(B, offset); } + +void Assembler::bl(int32_t offset) { GenB(BL, offset); } + +void Assembler::beq(Register rj, Register rd, int32_t offset) { + GenBJ(BEQ, rj, rd, offset); +} + +void Assembler::bne(Register rj, Register rd, int32_t offset) { + GenBJ(BNE, rj, rd, offset); +} + +void Assembler::blt(Register rj, Register rd, int32_t offset) { + GenBJ(BLT, rj, rd, offset); +} + +void Assembler::bge(Register rj, Register rd, int32_t offset) { + GenBJ(BGE, rj, rd, offset); +} + +void Assembler::bltu(Register rj, Register rd, int32_t offset) { + GenBJ(BLTU, rj, rd, offset); +} + +void Assembler::bgeu(Register rj, Register rd, int32_t offset) { + GenBJ(BGEU, rj, rd, offset); +} + +void Assembler::beqz(Register rj, int32_t offset) { GenB(BEQZ, rj, offset); } +void Assembler::bnez(Register rj, int32_t offset) { GenB(BNEZ, rj, offset); } + +void Assembler::jirl(Register rd, Register rj, int32_t offset) { + GenBJ(JIRL, rj, rd, offset); +} + +void Assembler::bceqz(CFRegister cj, int32_t si21) { + GenB(BCZ, cj, si21, true); +} + +void Assembler::bcnez(CFRegister cj, int32_t si21) { + GenB(BCZ, cj, si21, false); +} + +// -------Data-processing-instructions--------- + +// Arithmetic. +void Assembler::add_w(Register rd, Register rj, Register rk) { + GenRegister(ADD_W, rk, rj, rd); +} + +void Assembler::add_d(Register rd, Register rj, Register rk) { + GenRegister(ADD_D, rk, rj, rd); +} + +void Assembler::sub_w(Register rd, Register rj, Register rk) { + GenRegister(SUB_W, rk, rj, rd); +} + +void Assembler::sub_d(Register rd, Register rj, Register rk) { + GenRegister(SUB_D, rk, rj, rd); +} + +void Assembler::addi_w(Register rd, Register rj, int32_t si12) { + GenImm(ADDI_W, si12, rj, rd, 12); +} + +void Assembler::addi_d(Register rd, Register rj, int32_t si12) { + GenImm(ADDI_D, si12, rj, rd, 12); +} + +void Assembler::addu16i_d(Register rd, Register rj, int32_t si16) { + GenImm(ADDU16I_D, si16, rj, rd, 16); +} + +void Assembler::alsl_w(Register rd, Register rj, Register rk, int32_t sa2) { + DCHECK(is_uint2(sa2 - 1)); + GenImm(ALSL_W, sa2 - 1, rk, rj, rd); +} + +void Assembler::alsl_wu(Register rd, Register rj, Register rk, int32_t sa2) { + DCHECK(is_uint2(sa2 - 1)); + GenImm(ALSL_WU, sa2 + 3, rk, rj, rd); +} + +void Assembler::alsl_d(Register rd, Register rj, Register rk, int32_t sa2) { + DCHECK(is_uint2(sa2 - 1)); + GenImm(ALSL_D, sa2 - 1, rk, rj, rd); +} + +void Assembler::lu12i_w(Register rd, int32_t si20) { + GenImm(LU12I_W, si20, rd); +} + +void Assembler::lu32i_d(Register rd, int32_t si20) { + GenImm(LU32I_D, si20, rd); +} + +void Assembler::lu52i_d(Register rd, Register rj, int32_t si12) { + GenImm(LU52I_D, si12, rj, rd, 12); +} + +void Assembler::slt(Register rd, Register rj, Register rk) { + GenRegister(SLT, rk, rj, rd); +} + +void Assembler::sltu(Register rd, Register rj, Register rk) { + GenRegister(SLTU, rk, rj, rd); +} + +void Assembler::slti(Register rd, Register rj, int32_t si12) { + GenImm(SLTI, si12, rj, rd, 12); +} + +void Assembler::sltui(Register rd, Register rj, int32_t si12) { + GenImm(SLTUI, si12, rj, rd, 12); +} + +void Assembler::pcaddi(Register rd, int32_t si20) { GenImm(PCADDI, si20, rd); } + +void Assembler::pcaddu12i(Register rd, int32_t si20) { + GenImm(PCADDU12I, si20, rd); +} + +void Assembler::pcaddu18i(Register rd, int32_t si20) { + GenImm(PCADDU18I, si20, rd); +} + +void Assembler::pcalau12i(Register rd, int32_t si20) { + GenImm(PCALAU12I, si20, rd); +} + +void Assembler::and_(Register rd, Register rj, Register rk) { + GenRegister(AND, rk, rj, rd); +} + +void Assembler::or_(Register rd, Register rj, Register rk) { + GenRegister(OR, rk, rj, rd); +} + +void Assembler::xor_(Register rd, Register rj, Register rk) { + GenRegister(XOR, rk, rj, rd); +} + +void Assembler::nor(Register rd, Register rj, Register rk) { + GenRegister(NOR, rk, rj, rd); +} + +void Assembler::andn(Register rd, Register rj, Register rk) { + GenRegister(ANDN, rk, rj, rd); +} + +void Assembler::orn(Register rd, Register rj, Register rk) { + GenRegister(ORN, rk, rj, rd); +} + +void Assembler::andi(Register rd, Register rj, int32_t ui12) { + GenImm(ANDI, ui12, rj, rd, 12); +} + +void Assembler::ori(Register rd, Register rj, int32_t ui12) { + GenImm(ORI, ui12, rj, rd, 12); +} + +void Assembler::xori(Register rd, Register rj, int32_t ui12) { + GenImm(XORI, ui12, rj, rd, 12); +} + +void Assembler::mul_w(Register rd, Register rj, Register rk) { + GenRegister(MUL_W, rk, rj, rd); +} + +void Assembler::mulh_w(Register rd, Register rj, Register rk) { + GenRegister(MULH_W, rk, rj, rd); +} + +void Assembler::mulh_wu(Register rd, Register rj, Register rk) { + GenRegister(MULH_WU, rk, rj, rd); +} + +void Assembler::mul_d(Register rd, Register rj, Register rk) { + GenRegister(MUL_D, rk, rj, rd); +} + +void Assembler::mulh_d(Register rd, Register rj, Register rk) { + GenRegister(MULH_D, rk, rj, rd); +} + +void Assembler::mulh_du(Register rd, Register rj, Register rk) { + GenRegister(MULH_DU, rk, rj, rd); +} + +void Assembler::mulw_d_w(Register rd, Register rj, Register rk) { + GenRegister(MULW_D_W, rk, rj, rd); +} + +void Assembler::mulw_d_wu(Register rd, Register rj, Register rk) { + GenRegister(MULW_D_WU, rk, rj, rd); +} + +void Assembler::div_w(Register rd, Register rj, Register rk) { + GenRegister(DIV_W, rk, rj, rd); +} + +void Assembler::mod_w(Register rd, Register rj, Register rk) { + GenRegister(MOD_W, rk, rj, rd); +} + +void Assembler::div_wu(Register rd, Register rj, Register rk) { + GenRegister(DIV_WU, rk, rj, rd); +} + +void Assembler::mod_wu(Register rd, Register rj, Register rk) { + GenRegister(MOD_WU, rk, rj, rd); +} + +void Assembler::div_d(Register rd, Register rj, Register rk) { + GenRegister(DIV_D, rk, rj, rd); +} + +void Assembler::mod_d(Register rd, Register rj, Register rk) { + GenRegister(MOD_D, rk, rj, rd); +} + +void Assembler::div_du(Register rd, Register rj, Register rk) { + GenRegister(DIV_DU, rk, rj, rd); +} + +void Assembler::mod_du(Register rd, Register rj, Register rk) { + GenRegister(MOD_DU, rk, rj, rd); +} + +// Shifts. +void Assembler::sll_w(Register rd, Register rj, Register rk) { + GenRegister(SLL_W, rk, rj, rd); +} + +void Assembler::srl_w(Register rd, Register rj, Register rk) { + GenRegister(SRL_W, rk, rj, rd); +} + +void Assembler::sra_w(Register rd, Register rj, Register rk) { + GenRegister(SRA_W, rk, rj, rd); +} + +void Assembler::rotr_w(Register rd, Register rj, Register rk) { + GenRegister(ROTR_W, rk, rj, rd); +} + +void Assembler::slli_w(Register rd, Register rj, int32_t ui5) { + DCHECK(is_uint5(ui5)); + GenImm(SLLI_W, ui5 + 0x20, rj, rd, 6); +} + +void Assembler::srli_w(Register rd, Register rj, int32_t ui5) { + DCHECK(is_uint5(ui5)); + GenImm(SRLI_W, ui5 + 0x20, rj, rd, 6); +} + +void Assembler::srai_w(Register rd, Register rj, int32_t ui5) { + DCHECK(is_uint5(ui5)); + GenImm(SRAI_W, ui5 + 0x20, rj, rd, 6); +} + +void Assembler::rotri_w(Register rd, Register rj, int32_t ui5) { + DCHECK(is_uint5(ui5)); + GenImm(ROTRI_W, ui5 + 0x20, rj, rd, 6); +} + +void Assembler::sll_d(Register rd, Register rj, Register rk) { + GenRegister(SLL_D, rk, rj, rd); +} + +void Assembler::srl_d(Register rd, Register rj, Register rk) { + GenRegister(SRL_D, rk, rj, rd); +} + +void Assembler::sra_d(Register rd, Register rj, Register rk) { + GenRegister(SRA_D, rk, rj, rd); +} + +void Assembler::rotr_d(Register rd, Register rj, Register rk) { + GenRegister(ROTR_D, rk, rj, rd); +} + +void Assembler::slli_d(Register rd, Register rj, int32_t ui6) { + GenImm(SLLI_D, ui6, rj, rd, 6); +} + +void Assembler::srli_d(Register rd, Register rj, int32_t ui6) { + GenImm(SRLI_D, ui6, rj, rd, 6); +} + +void Assembler::srai_d(Register rd, Register rj, int32_t ui6) { + GenImm(SRAI_D, ui6, rj, rd, 6); +} + +void Assembler::rotri_d(Register rd, Register rj, int32_t ui6) { + GenImm(ROTRI_D, ui6, rj, rd, 6); +} + +// Bit twiddling. +void Assembler::ext_w_b(Register rd, Register rj) { + GenRegister(EXT_W_B, rj, rd); +} + +void Assembler::ext_w_h(Register rd, Register rj) { + GenRegister(EXT_W_H, rj, rd); +} + +void Assembler::clo_w(Register rd, Register rj) { GenRegister(CLO_W, rj, rd); } + +void Assembler::clz_w(Register rd, Register rj) { GenRegister(CLZ_W, rj, rd); } + +void Assembler::cto_w(Register rd, Register rj) { GenRegister(CTO_W, rj, rd); } + +void Assembler::ctz_w(Register rd, Register rj) { GenRegister(CTZ_W, rj, rd); } + +void Assembler::clo_d(Register rd, Register rj) { GenRegister(CLO_D, rj, rd); } + +void Assembler::clz_d(Register rd, Register rj) { GenRegister(CLZ_D, rj, rd); } + +void Assembler::cto_d(Register rd, Register rj) { GenRegister(CTO_D, rj, rd); } + +void Assembler::ctz_d(Register rd, Register rj) { GenRegister(CTZ_D, rj, rd); } + +void Assembler::bytepick_w(Register rd, Register rj, Register rk, int32_t sa2) { + DCHECK(is_uint2(sa2)); + GenImm(BYTEPICK_W, sa2, rk, rj, rd); +} + +void Assembler::bytepick_d(Register rd, Register rj, Register rk, int32_t sa3) { + GenImm(BYTEPICK_D, sa3, rk, rj, rd); +} + +void Assembler::revb_2h(Register rd, Register rj) { + GenRegister(REVB_2H, rj, rd); +} + +void Assembler::revb_4h(Register rd, Register rj) { + GenRegister(REVB_4H, rj, rd); +} + +void Assembler::revb_2w(Register rd, Register rj) { + GenRegister(REVB_2W, rj, rd); +} + +void Assembler::revb_d(Register rd, Register rj) { + GenRegister(REVB_D, rj, rd); +} + +void Assembler::revh_2w(Register rd, Register rj) { + GenRegister(REVH_2W, rj, rd); +} + +void Assembler::revh_d(Register rd, Register rj) { + GenRegister(REVH_D, rj, rd); +} + +void Assembler::bitrev_4b(Register rd, Register rj) { + GenRegister(BITREV_4B, rj, rd); +} + +void Assembler::bitrev_8b(Register rd, Register rj) { + GenRegister(BITREV_8B, rj, rd); +} + +void Assembler::bitrev_w(Register rd, Register rj) { + GenRegister(BITREV_W, rj, rd); +} + +void Assembler::bitrev_d(Register rd, Register rj) { + GenRegister(BITREV_D, rj, rd); +} + +void Assembler::bstrins_w(Register rd, Register rj, int32_t msbw, + int32_t lsbw) { + DCHECK(is_uint5(msbw) && is_uint5(lsbw)); + GenImm(BSTR_W, msbw + 0x20, lsbw, rj, rd); +} + +void Assembler::bstrins_d(Register rd, Register rj, int32_t msbd, + int32_t lsbd) { + GenImm(BSTRINS_D, msbd, lsbd, rj, rd); +} + +void Assembler::bstrpick_w(Register rd, Register rj, int32_t msbw, + int32_t lsbw) { + DCHECK(is_uint5(msbw) && is_uint5(lsbw)); + GenImm(BSTR_W, msbw + 0x20, lsbw + 0x20, rj, rd); +} + +void Assembler::bstrpick_d(Register rd, Register rj, int32_t msbd, + int32_t lsbd) { + GenImm(BSTRPICK_D, msbd, lsbd, rj, rd); +} + +void Assembler::maskeqz(Register rd, Register rj, Register rk) { + GenRegister(MASKEQZ, rk, rj, rd); +} + +void Assembler::masknez(Register rd, Register rj, Register rk) { + GenRegister(MASKNEZ, rk, rj, rd); +} + +// Memory-instructions +void Assembler::ld_b(Register rd, Register rj, int32_t si12) { + GenImm(LD_B, si12, rj, rd, 12); +} + +void Assembler::ld_h(Register rd, Register rj, int32_t si12) { + GenImm(LD_H, si12, rj, rd, 12); +} + +void Assembler::ld_w(Register rd, Register rj, int32_t si12) { + GenImm(LD_W, si12, rj, rd, 12); +} + +void Assembler::ld_d(Register rd, Register rj, int32_t si12) { + GenImm(LD_D, si12, rj, rd, 12); +} + +void Assembler::ld_bu(Register rd, Register rj, int32_t si12) { + GenImm(LD_BU, si12, rj, rd, 12); +} + +void Assembler::ld_hu(Register rd, Register rj, int32_t si12) { + GenImm(LD_HU, si12, rj, rd, 12); +} + +void Assembler::ld_wu(Register rd, Register rj, int32_t si12) { + GenImm(LD_WU, si12, rj, rd, 12); +} + +void Assembler::st_b(Register rd, Register rj, int32_t si12) { + GenImm(ST_B, si12, rj, rd, 12); +} + +void Assembler::st_h(Register rd, Register rj, int32_t si12) { + GenImm(ST_H, si12, rj, rd, 12); +} + +void Assembler::st_w(Register rd, Register rj, int32_t si12) { + GenImm(ST_W, si12, rj, rd, 12); +} + +void Assembler::st_d(Register rd, Register rj, int32_t si12) { + GenImm(ST_D, si12, rj, rd, 12); +} + +void Assembler::ldx_b(Register rd, Register rj, Register rk) { + GenRegister(LDX_B, rk, rj, rd); +} + +void Assembler::ldx_h(Register rd, Register rj, Register rk) { + GenRegister(LDX_H, rk, rj, rd); +} + +void Assembler::ldx_w(Register rd, Register rj, Register rk) { + GenRegister(LDX_W, rk, rj, rd); +} + +void Assembler::ldx_d(Register rd, Register rj, Register rk) { + GenRegister(LDX_D, rk, rj, rd); +} + +void Assembler::ldx_bu(Register rd, Register rj, Register rk) { + GenRegister(LDX_BU, rk, rj, rd); +} + +void Assembler::ldx_hu(Register rd, Register rj, Register rk) { + GenRegister(LDX_HU, rk, rj, rd); +} + +void Assembler::ldx_wu(Register rd, Register rj, Register rk) { + GenRegister(LDX_WU, rk, rj, rd); +} + +void Assembler::stx_b(Register rd, Register rj, Register rk) { + GenRegister(STX_B, rk, rj, rd); +} + +void Assembler::stx_h(Register rd, Register rj, Register rk) { + GenRegister(STX_H, rk, rj, rd); +} + +void Assembler::stx_w(Register rd, Register rj, Register rk) { + GenRegister(STX_W, rk, rj, rd); +} + +void Assembler::stx_d(Register rd, Register rj, Register rk) { + GenRegister(STX_D, rk, rj, rd); +} + +void Assembler::ldptr_w(Register rd, Register rj, int32_t si14) { + DCHECK(is_int16(si14) && ((si14 & 0x3) == 0)); + GenImm(LDPTR_W, si14 >> 2, rj, rd, 14); +} + +void Assembler::ldptr_d(Register rd, Register rj, int32_t si14) { + DCHECK(is_int16(si14) && ((si14 & 0x3) == 0)); + GenImm(LDPTR_D, si14 >> 2, rj, rd, 14); +} + +void Assembler::stptr_w(Register rd, Register rj, int32_t si14) { + DCHECK(is_int16(si14) && ((si14 & 0x3) == 0)); + GenImm(STPTR_W, si14 >> 2, rj, rd, 14); +} + +void Assembler::stptr_d(Register rd, Register rj, int32_t si14) { + DCHECK(is_int16(si14) && ((si14 & 0x3) == 0)); + GenImm(STPTR_D, si14 >> 2, rj, rd, 14); +} + +void Assembler::amswap_w(Register rd, Register rk, Register rj) { + GenRegister(AMSWAP_W, rk, rj, rd); +} + +void Assembler::amswap_d(Register rd, Register rk, Register rj) { + GenRegister(AMSWAP_D, rk, rj, rd); +} + +void Assembler::amadd_w(Register rd, Register rk, Register rj) { + GenRegister(AMADD_W, rk, rj, rd); +} + +void Assembler::amadd_d(Register rd, Register rk, Register rj) { + GenRegister(AMADD_D, rk, rj, rd); +} + +void Assembler::amand_w(Register rd, Register rk, Register rj) { + GenRegister(AMAND_W, rk, rj, rd); +} + +void Assembler::amand_d(Register rd, Register rk, Register rj) { + GenRegister(AMAND_D, rk, rj, rd); +} + +void Assembler::amor_w(Register rd, Register rk, Register rj) { + GenRegister(AMOR_W, rk, rj, rd); +} + +void Assembler::amor_d(Register rd, Register rk, Register rj) { + GenRegister(AMOR_D, rk, rj, rd); +} + +void Assembler::amxor_w(Register rd, Register rk, Register rj) { + GenRegister(AMXOR_W, rk, rj, rd); +} + +void Assembler::amxor_d(Register rd, Register rk, Register rj) { + GenRegister(AMXOR_D, rk, rj, rd); +} + +void Assembler::ammax_w(Register rd, Register rk, Register rj) { + GenRegister(AMMAX_W, rk, rj, rd); +} + +void Assembler::ammax_d(Register rd, Register rk, Register rj) { + GenRegister(AMMAX_D, rk, rj, rd); +} + +void Assembler::ammin_w(Register rd, Register rk, Register rj) { + GenRegister(AMMIN_W, rk, rj, rd); +} + +void Assembler::ammin_d(Register rd, Register rk, Register rj) { + GenRegister(AMMIN_D, rk, rj, rd); +} + +void Assembler::ammax_wu(Register rd, Register rk, Register rj) { + GenRegister(AMMAX_WU, rk, rj, rd); +} + +void Assembler::ammax_du(Register rd, Register rk, Register rj) { + GenRegister(AMMAX_DU, rk, rj, rd); +} + +void Assembler::ammin_wu(Register rd, Register rk, Register rj) { + GenRegister(AMMIN_WU, rk, rj, rd); +} + +void Assembler::ammin_du(Register rd, Register rk, Register rj) { + GenRegister(AMMIN_DU, rk, rj, rd); +} + +void Assembler::amswap_db_w(Register rd, Register rk, Register rj) { + GenRegister(AMSWAP_DB_W, rk, rj, rd); +} + +void Assembler::amswap_db_d(Register rd, Register rk, Register rj) { + GenRegister(AMSWAP_DB_D, rk, rj, rd); +} + +void Assembler::amadd_db_w(Register rd, Register rk, Register rj) { + GenRegister(AMADD_DB_W, rk, rj, rd); +} + +void Assembler::amadd_db_d(Register rd, Register rk, Register rj) { + GenRegister(AMADD_DB_D, rk, rj, rd); +} + +void Assembler::amand_db_w(Register rd, Register rk, Register rj) { + GenRegister(AMAND_DB_W, rk, rj, rd); +} + +void Assembler::amand_db_d(Register rd, Register rk, Register rj) { + GenRegister(AMAND_DB_D, rk, rj, rd); +} + +void Assembler::amor_db_w(Register rd, Register rk, Register rj) { + GenRegister(AMOR_DB_W, rk, rj, rd); +} + +void Assembler::amor_db_d(Register rd, Register rk, Register rj) { + GenRegister(AMOR_DB_D, rk, rj, rd); +} + +void Assembler::amxor_db_w(Register rd, Register rk, Register rj) { + GenRegister(AMXOR_DB_W, rk, rj, rd); +} + +void Assembler::amxor_db_d(Register rd, Register rk, Register rj) { + GenRegister(AMXOR_DB_D, rk, rj, rd); +} + +void Assembler::ammax_db_w(Register rd, Register rk, Register rj) { + GenRegister(AMMAX_DB_W, rk, rj, rd); +} + +void Assembler::ammax_db_d(Register rd, Register rk, Register rj) { + GenRegister(AMMAX_DB_D, rk, rj, rd); +} + +void Assembler::ammin_db_w(Register rd, Register rk, Register rj) { + GenRegister(AMMIN_DB_W, rk, rj, rd); +} + +void Assembler::ammin_db_d(Register rd, Register rk, Register rj) { + GenRegister(AMMIN_DB_D, rk, rj, rd); +} + +void Assembler::ammax_db_wu(Register rd, Register rk, Register rj) { + GenRegister(AMMAX_DB_WU, rk, rj, rd); +} + +void Assembler::ammax_db_du(Register rd, Register rk, Register rj) { + GenRegister(AMMAX_DB_DU, rk, rj, rd); +} + +void Assembler::ammin_db_wu(Register rd, Register rk, Register rj) { + GenRegister(AMMIN_DB_WU, rk, rj, rd); +} + +void Assembler::ammin_db_du(Register rd, Register rk, Register rj) { + GenRegister(AMMIN_DB_DU, rk, rj, rd); +} + +void Assembler::ll_w(Register rd, Register rj, int32_t si14) { + DCHECK(is_int16(si14) && ((si14 & 0x3) == 0)); + GenImm(LL_W, si14 >> 2, rj, rd, 14); +} + +void Assembler::ll_d(Register rd, Register rj, int32_t si14) { + DCHECK(is_int16(si14) && ((si14 & 0x3) == 0)); + GenImm(LL_D, si14 >> 2, rj, rd, 14); +} + +void Assembler::sc_w(Register rd, Register rj, int32_t si14) { + DCHECK(is_int16(si14) && ((si14 & 0x3) == 0)); + GenImm(SC_W, si14 >> 2, rj, rd, 14); +} + +void Assembler::sc_d(Register rd, Register rj, int32_t si14) { + DCHECK(is_int16(si14) && ((si14 & 0x3) == 0)); + GenImm(SC_D, si14 >> 2, rj, rd, 14); +} + +void Assembler::dbar(int32_t hint) { GenImm(DBAR, hint); } + +void Assembler::ibar(int32_t hint) { GenImm(IBAR, hint); } + +// Break instruction. +void Assembler::break_(uint32_t code, bool break_as_stop) { + DCHECK( + (break_as_stop && code <= kMaxStopCode && code > kMaxWatchpointCode) || + (!break_as_stop && (code > kMaxStopCode || code <= kMaxWatchpointCode))); + GenImm(BREAK, code); +} + +void Assembler::stop(uint32_t code) { + DCHECK_GT(code, kMaxWatchpointCode); + DCHECK_LE(code, kMaxStopCode); +#if defined(V8_HOST_ARCH_LOONG64) + break_(0x4321); +#else // V8_HOST_ARCH_LOONG64 + break_(code, true); +#endif +} + +void Assembler::fadd_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FADD_S, fk, fj, fd); +} + +void Assembler::fadd_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FADD_D, fk, fj, fd); +} + +void Assembler::fsub_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FSUB_S, fk, fj, fd); +} + +void Assembler::fsub_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FSUB_D, fk, fj, fd); +} + +void Assembler::fmul_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMUL_S, fk, fj, fd); +} + +void Assembler::fmul_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMUL_D, fk, fj, fd); +} + +void Assembler::fdiv_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FDIV_S, fk, fj, fd); +} + +void Assembler::fdiv_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FDIV_D, fk, fj, fd); +} + +void Assembler::fmadd_s(FPURegister fd, FPURegister fj, FPURegister fk, + FPURegister fa) { + GenRegister(FMADD_S, fa, fk, fj, fd); +} + +void Assembler::fmadd_d(FPURegister fd, FPURegister fj, FPURegister fk, + FPURegister fa) { + GenRegister(FMADD_D, fa, fk, fj, fd); +} + +void Assembler::fmsub_s(FPURegister fd, FPURegister fj, FPURegister fk, + FPURegister fa) { + GenRegister(FMSUB_S, fa, fk, fj, fd); +} + +void Assembler::fmsub_d(FPURegister fd, FPURegister fj, FPURegister fk, + FPURegister fa) { + GenRegister(FMSUB_D, fa, fk, fj, fd); +} + +void Assembler::fnmadd_s(FPURegister fd, FPURegister fj, FPURegister fk, + FPURegister fa) { + GenRegister(FNMADD_S, fa, fk, fj, fd); +} + +void Assembler::fnmadd_d(FPURegister fd, FPURegister fj, FPURegister fk, + FPURegister fa) { + GenRegister(FNMADD_D, fa, fk, fj, fd); +} + +void Assembler::fnmsub_s(FPURegister fd, FPURegister fj, FPURegister fk, + FPURegister fa) { + GenRegister(FNMSUB_S, fa, fk, fj, fd); +} + +void Assembler::fnmsub_d(FPURegister fd, FPURegister fj, FPURegister fk, + FPURegister fa) { + GenRegister(FNMSUB_D, fa, fk, fj, fd); +} + +void Assembler::fmax_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMAX_S, fk, fj, fd); +} + +void Assembler::fmax_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMAX_D, fk, fj, fd); +} + +void Assembler::fmin_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMIN_S, fk, fj, fd); +} + +void Assembler::fmin_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMIN_D, fk, fj, fd); +} + +void Assembler::fmaxa_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMAXA_S, fk, fj, fd); +} + +void Assembler::fmaxa_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMAXA_D, fk, fj, fd); +} + +void Assembler::fmina_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMINA_S, fk, fj, fd); +} + +void Assembler::fmina_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FMINA_D, fk, fj, fd); +} + +void Assembler::fabs_s(FPURegister fd, FPURegister fj) { + GenRegister(FABS_S, fj, fd); +} + +void Assembler::fabs_d(FPURegister fd, FPURegister fj) { + GenRegister(FABS_D, fj, fd); +} + +void Assembler::fneg_s(FPURegister fd, FPURegister fj) { + GenRegister(FNEG_S, fj, fd); +} + +void Assembler::fneg_d(FPURegister fd, FPURegister fj) { + GenRegister(FNEG_D, fj, fd); +} + +void Assembler::fsqrt_s(FPURegister fd, FPURegister fj) { + GenRegister(FSQRT_S, fj, fd); +} + +void Assembler::fsqrt_d(FPURegister fd, FPURegister fj) { + GenRegister(FSQRT_D, fj, fd); +} + +void Assembler::frecip_s(FPURegister fd, FPURegister fj) { + GenRegister(FRECIP_S, fj, fd); +} + +void Assembler::frecip_d(FPURegister fd, FPURegister fj) { + GenRegister(FRECIP_D, fj, fd); +} + +void Assembler::frsqrt_s(FPURegister fd, FPURegister fj) { + GenRegister(FRSQRT_S, fj, fd); +} + +void Assembler::frsqrt_d(FPURegister fd, FPURegister fj) { + GenRegister(FRSQRT_D, fj, fd); +} + +void Assembler::fscaleb_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FSCALEB_S, fk, fj, fd); +} + +void Assembler::fscaleb_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FSCALEB_D, fk, fj, fd); +} + +void Assembler::flogb_s(FPURegister fd, FPURegister fj) { + GenRegister(FLOGB_S, fj, fd); +} + +void Assembler::flogb_d(FPURegister fd, FPURegister fj) { + GenRegister(FLOGB_D, fj, fd); +} + +void Assembler::fcopysign_s(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FCOPYSIGN_S, fk, fj, fd); +} + +void Assembler::fcopysign_d(FPURegister fd, FPURegister fj, FPURegister fk) { + GenRegister(FCOPYSIGN_D, fk, fj, fd); +} + +void Assembler::fclass_s(FPURegister fd, FPURegister fj) { + GenRegister(FCLASS_S, fj, fd); +} + +void Assembler::fclass_d(FPURegister fd, FPURegister fj) { + GenRegister(FCLASS_D, fj, fd); +} + +void Assembler::fcmp_cond_s(FPUCondition cc, FPURegister fj, FPURegister fk, + CFRegister cd) { + GenCmp(FCMP_COND_S, cc, fk, fj, cd); +} + +void Assembler::fcmp_cond_d(FPUCondition cc, FPURegister fj, FPURegister fk, + CFRegister cd) { + GenCmp(FCMP_COND_D, cc, fk, fj, cd); +} + +void Assembler::fcvt_s_d(FPURegister fd, FPURegister fj) { + GenRegister(FCVT_S_D, fj, fd); +} + +void Assembler::fcvt_d_s(FPURegister fd, FPURegister fj) { + GenRegister(FCVT_D_S, fj, fd); +} + +void Assembler::ffint_s_w(FPURegister fd, FPURegister fj) { + GenRegister(FFINT_S_W, fj, fd); +} + +void Assembler::ffint_s_l(FPURegister fd, FPURegister fj) { + GenRegister(FFINT_S_L, fj, fd); +} + +void Assembler::ffint_d_w(FPURegister fd, FPURegister fj) { + GenRegister(FFINT_D_W, fj, fd); +} + +void Assembler::ffint_d_l(FPURegister fd, FPURegister fj) { + GenRegister(FFINT_D_L, fj, fd); +} + +void Assembler::ftint_w_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINT_W_S, fj, fd); +} + +void Assembler::ftint_w_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINT_W_D, fj, fd); +} + +void Assembler::ftint_l_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINT_L_S, fj, fd); +} + +void Assembler::ftint_l_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINT_L_D, fj, fd); +} + +void Assembler::ftintrm_w_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRM_W_S, fj, fd); +} + +void Assembler::ftintrm_w_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRM_W_D, fj, fd); +} + +void Assembler::ftintrm_l_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRM_L_S, fj, fd); +} + +void Assembler::ftintrm_l_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRM_L_D, fj, fd); +} + +void Assembler::ftintrp_w_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRP_W_S, fj, fd); +} + +void Assembler::ftintrp_w_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRP_W_D, fj, fd); +} + +void Assembler::ftintrp_l_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRP_L_S, fj, fd); +} + +void Assembler::ftintrp_l_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRP_L_D, fj, fd); +} + +void Assembler::ftintrz_w_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRZ_W_S, fj, fd); +} + +void Assembler::ftintrz_w_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRZ_W_D, fj, fd); +} + +void Assembler::ftintrz_l_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRZ_L_S, fj, fd); +} + +void Assembler::ftintrz_l_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRZ_L_D, fj, fd); +} + +void Assembler::ftintrne_w_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRNE_W_S, fj, fd); +} + +void Assembler::ftintrne_w_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRNE_W_D, fj, fd); +} + +void Assembler::ftintrne_l_s(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRNE_L_S, fj, fd); +} + +void Assembler::ftintrne_l_d(FPURegister fd, FPURegister fj) { + GenRegister(FTINTRNE_L_D, fj, fd); +} + +void Assembler::frint_s(FPURegister fd, FPURegister fj) { + GenRegister(FRINT_S, fj, fd); +} + +void Assembler::frint_d(FPURegister fd, FPURegister fj) { + GenRegister(FRINT_D, fj, fd); +} + +void Assembler::fmov_s(FPURegister fd, FPURegister fj) { + GenRegister(FMOV_S, fj, fd); +} + +void Assembler::fmov_d(FPURegister fd, FPURegister fj) { + GenRegister(FMOV_D, fj, fd); +} + +void Assembler::fsel(CFRegister ca, FPURegister fd, FPURegister fj, + FPURegister fk) { + GenSel(FSEL, ca, fk, fj, fd); +} + +void Assembler::movgr2fr_w(FPURegister fd, Register rj) { + GenRegister(MOVGR2FR_W, rj, fd); +} + +void Assembler::movgr2fr_d(FPURegister fd, Register rj) { + GenRegister(MOVGR2FR_D, rj, fd); +} + +void Assembler::movgr2frh_w(FPURegister fd, Register rj) { + GenRegister(MOVGR2FRH_W, rj, fd); +} + +void Assembler::movfr2gr_s(Register rd, FPURegister fj) { + GenRegister(MOVFR2GR_S, fj, rd); +} + +void Assembler::movfr2gr_d(Register rd, FPURegister fj) { + GenRegister(MOVFR2GR_D, fj, rd); +} + +void Assembler::movfrh2gr_s(Register rd, FPURegister fj) { + GenRegister(MOVFRH2GR_S, fj, rd); +} + +void Assembler::movgr2fcsr(Register rj, FPUControlRegister fcsr) { + GenRegister(MOVGR2FCSR, rj, fcsr); +} + +void Assembler::movfcsr2gr(Register rd, FPUControlRegister fcsr) { + GenRegister(MOVFCSR2GR, fcsr, rd); +} + +void Assembler::movfr2cf(CFRegister cd, FPURegister fj) { + GenRegister(MOVFR2CF, fj, cd); +} + +void Assembler::movcf2fr(FPURegister fd, CFRegister cj) { + GenRegister(MOVCF2FR, cj, fd); +} + +void Assembler::movgr2cf(CFRegister cd, Register rj) { + GenRegister(MOVGR2CF, rj, cd); +} + +void Assembler::movcf2gr(Register rd, CFRegister cj) { + GenRegister(MOVCF2GR, cj, rd); +} + +void Assembler::fld_s(FPURegister fd, Register rj, int32_t si12) { + GenImm(FLD_S, si12, rj, fd); +} + +void Assembler::fld_d(FPURegister fd, Register rj, int32_t si12) { + GenImm(FLD_D, si12, rj, fd); +} + +void Assembler::fst_s(FPURegister fd, Register rj, int32_t si12) { + GenImm(FST_S, si12, rj, fd); +} + +void Assembler::fst_d(FPURegister fd, Register rj, int32_t si12) { + GenImm(FST_D, si12, rj, fd); +} + +void Assembler::fldx_s(FPURegister fd, Register rj, Register rk) { + GenRegister(FLDX_S, rk, rj, fd); +} + +void Assembler::fldx_d(FPURegister fd, Register rj, Register rk) { + GenRegister(FLDX_D, rk, rj, fd); +} + +void Assembler::fstx_s(FPURegister fd, Register rj, Register rk) { + GenRegister(FSTX_S, rk, rj, fd); +} + +void Assembler::fstx_d(FPURegister fd, Register rj, Register rk) { + GenRegister(FSTX_D, rk, rj, fd); +} + +void Assembler::AdjustBaseAndOffset(MemOperand* src) { + // is_int12 must be passed a signed value, hence the static cast below. + if ((!src->hasIndexReg() && is_int12(src->offset())) || src->hasIndexReg()) { + return; + } + UseScratchRegisterScope temps(this); + Register scratch = temps.Acquire(); + if (is_uint12(static_cast<int32_t>(src->offset()))) { + ori(scratch, zero_reg, src->offset() & kImm12Mask); + } else { + lu12i_w(scratch, src->offset() >> 12 & 0xfffff); + if (src->offset() & kImm12Mask) { + ori(scratch, scratch, src->offset() & kImm12Mask); + } + } + src->index_ = scratch; + src->offset_ = 0; +} + +int Assembler::RelocateInternalReference(RelocInfo::Mode rmode, Address pc, + intptr_t pc_delta) { + DCHECK(RelocInfo::IsInternalReference(rmode)); + int64_t* p = reinterpret_cast<int64_t*>(pc); + if (*p == kEndOfJumpChain) { + return 0; // Number of instructions patched. + } + *p += pc_delta; + return 2; // Number of instructions patched. +} + +void Assembler::RelocateRelativeReference(RelocInfo::Mode rmode, Address pc, + intptr_t pc_delta) { + DCHECK(RelocInfo::IsRelativeCodeTarget(rmode)); + Instr instr = instr_at(pc); + int32_t offset = instr & kImm26Mask; + offset = (((offset & 0x3ff) << 22 >> 6) | ((offset >> 10) & kImm16Mask)) << 2; + offset -= pc_delta; + uint32_t* p = reinterpret_cast<uint32_t*>(pc); + offset >>= 2; + offset = ((offset & kImm16Mask) << kRkShift) | ((offset & kImm26Mask) >> 16); + *p = (instr & ~kImm26Mask) | offset; + return; +} + +void Assembler::FixOnHeapReferences(bool update_embedded_objects) { + if (!update_embedded_objects) return; + for (auto p : saved_handles_for_raw_object_ptr_) { + Address address = reinterpret_cast<Address>(buffer_->start() + p.first); + Handle<HeapObject> object(reinterpret_cast<Address*>(p.second)); + set_target_value_at(address, object->ptr()); + } +} + +void Assembler::FixOnHeapReferencesToHandles() { + for (auto p : saved_handles_for_raw_object_ptr_) { + Address address = reinterpret_cast<Address>(buffer_->start() + p.first); + set_target_value_at(address, p.second); + } + saved_handles_for_raw_object_ptr_.clear(); +} + +void Assembler::GrowBuffer() { + bool previously_on_heap = buffer_->IsOnHeap(); + int previous_on_heap_gc_count = OnHeapGCCount(); + + // Compute new buffer size. + int old_size = buffer_->size(); + int new_size = std::min(2 * old_size, old_size + 1 * MB); + + // Some internal data structures overflow for very large buffers, + // they must ensure that kMaximalBufferSize is not too large. + if (new_size > kMaximalBufferSize) { + V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer"); + } + + // Set up new buffer. + std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size); + DCHECK_EQ(new_size, new_buffer->size()); + byte* new_start = new_buffer->start(); + + // Copy the data. + intptr_t pc_delta = new_start - buffer_start_; + intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size); + size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos(); + MemMove(new_start, buffer_start_, pc_offset()); + MemMove(reloc_info_writer.pos() + rc_delta, reloc_info_writer.pos(), + reloc_size); + + // Switch buffers. + buffer_ = std::move(new_buffer); + buffer_start_ = new_start; + pc_ += pc_delta; + last_call_pc_ += pc_delta; + reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta, + reloc_info_writer.last_pc() + pc_delta); + + // None of our relocation types are pc relative pointing outside the code + // buffer nor pc absolute pointing inside the code buffer, so there is no need + // to relocate any emitted relocation entries. + + // Relocate internal references. + for (auto pos : internal_reference_positions_) { + Address address = reinterpret_cast<intptr_t>(buffer_start_) + pos; + intptr_t internal_ref = ReadUnalignedValue<intptr_t>(address); + if (internal_ref != kEndOfJumpChain) { + internal_ref += pc_delta; + WriteUnalignedValue<intptr_t>(address, internal_ref); + } + } + + // Fix on-heap references. + if (previously_on_heap) { + if (buffer_->IsOnHeap()) { + FixOnHeapReferences(previous_on_heap_gc_count != OnHeapGCCount()); + } else { + FixOnHeapReferencesToHandles(); + } + } +} + +void Assembler::db(uint8_t data) { + if (!is_buffer_growth_blocked()) { + CheckBuffer(); + } + *reinterpret_cast<uint8_t*>(pc_) = data; + pc_ += sizeof(uint8_t); +} + +void Assembler::dd(uint32_t data, RelocInfo::Mode rmode) { + if (!is_buffer_growth_blocked()) { + CheckBuffer(); + } + if (!RelocInfo::IsNone(rmode)) { + DCHECK(RelocInfo::IsDataEmbeddedObject(rmode) || + RelocInfo::IsLiteralConstant(rmode)); + RecordRelocInfo(rmode); + } + *reinterpret_cast<uint32_t*>(pc_) = data; + pc_ += sizeof(uint32_t); +} + +void Assembler::dq(uint64_t data, RelocInfo::Mode rmode) { + if (!is_buffer_growth_blocked()) { + CheckBuffer(); + } + if (!RelocInfo::IsNone(rmode)) { + DCHECK(RelocInfo::IsDataEmbeddedObject(rmode) || + RelocInfo::IsLiteralConstant(rmode)); + RecordRelocInfo(rmode); + } + *reinterpret_cast<uint64_t*>(pc_) = data; + pc_ += sizeof(uint64_t); +} + +void Assembler::dd(Label* label) { + if (!is_buffer_growth_blocked()) { + CheckBuffer(); + } + uint64_t data; + if (label->is_bound()) { + data = reinterpret_cast<uint64_t>(buffer_start_ + label->pos()); + } else { + data = jump_address(label); + unbound_labels_count_++; + internal_reference_positions_.insert(label->pos()); + } + RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE); + EmitHelper(data); +} + +void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) { + if (!ShouldRecordRelocInfo(rmode)) return; + // We do not try to reuse pool constants. + RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code()); + DCHECK_GE(buffer_space(), kMaxRelocSize); // Too late to grow buffer here. + reloc_info_writer.Write(&rinfo); +} + +void Assembler::BlockTrampolinePoolFor(int instructions) { + CheckTrampolinePoolQuick(instructions); + BlockTrampolinePoolBefore(pc_offset() + instructions * kInstrSize); +} + +void Assembler::CheckTrampolinePool() { + // Some small sequences of instructions must not be broken up by the + // insertion of a trampoline pool; such sequences are protected by setting + // either trampoline_pool_blocked_nesting_ or no_trampoline_pool_before_, + // which are both checked here. Also, recursive calls to CheckTrampolinePool + // are blocked by trampoline_pool_blocked_nesting_. + if ((trampoline_pool_blocked_nesting_ > 0) || + (pc_offset() < no_trampoline_pool_before_)) { + // Emission is currently blocked; make sure we try again as soon as + // possible. + if (trampoline_pool_blocked_nesting_ > 0) { + next_buffer_check_ = pc_offset() + kInstrSize; + } else { + next_buffer_check_ = no_trampoline_pool_before_; + } + return; + } + + DCHECK(!trampoline_emitted_); + DCHECK_GE(unbound_labels_count_, 0); + if (unbound_labels_count_ > 0) { + // First we emit jump (2 instructions), then we emit trampoline pool. + { + BlockTrampolinePoolScope block_trampoline_pool(this); + Label after_pool; + b(&after_pool); + nop(); // TODO(LOONG_dev): remove this + + int pool_start = pc_offset(); + for (int i = 0; i < unbound_labels_count_; i++) { + { + b(&after_pool); + nop(); // TODO(LOONG_dev): remove this + } + } + nop(); + trampoline_ = Trampoline(pool_start, unbound_labels_count_); + bind(&after_pool); + + trampoline_emitted_ = true; + // As we are only going to emit trampoline once, we need to prevent any + // further emission. + next_buffer_check_ = kMaxInt; + } + } else { + // Number of branches to unbound label at this point is zero, so we can + // move next buffer check to maximum. + next_buffer_check_ = + pc_offset() + kMax16BranchOffset - kTrampolineSlotsSize * 16; + } + return; +} + +Address Assembler::target_address_at(Address pc) { + Instr instr0 = instr_at(pc); + if (IsB(instr0)) { + int32_t offset = instr0 & kImm26Mask; + offset = (((offset & 0x3ff) << 22 >> 6) | ((offset >> 10) & kImm16Mask)) + << 2; + return pc + offset; + } + Instr instr1 = instr_at(pc + 1 * kInstrSize); + Instr instr2 = instr_at(pc + 2 * kInstrSize); + + // Interpret 4 instructions for address generated by li: See listing in + // Assembler::set_target_address_at() just below. + DCHECK((IsLu12i_w(instr0) && (IsOri(instr1)) && (IsLu32i_d(instr2)))); + + // Assemble the 48 bit value. + uint64_t hi20 = ((uint64_t)(instr2 >> 5) & 0xfffff) << 32; + uint64_t mid20 = ((uint64_t)(instr0 >> 5) & 0xfffff) << 12; + uint64_t low12 = ((uint64_t)(instr1 >> 10) & 0xfff); + int64_t addr = static_cast<int64_t>(hi20 | mid20 | low12); + + // Sign extend to get canonical address. + addr = (addr << 16) >> 16; + return static_cast<Address>(addr); +} + +// On loong64, a target address is stored in a 3-instruction sequence: +// 0: lu12i_w(rd, (j.imm64_ >> 12) & kImm20Mask); +// 1: ori(rd, rd, j.imm64_ & kImm12Mask); +// 2: lu32i_d(rd, (j.imm64_ >> 32) & kImm20Mask); +// +// Patching the address must replace all the lui & ori instructions, +// and flush the i-cache. +// +void Assembler::set_target_value_at(Address pc, uint64_t target, + ICacheFlushMode icache_flush_mode) { + // There is an optimization where only 3 instructions are used to load address + // in code on LOONG64 because only 48-bits of address is effectively used. + // It relies on fact the upper [63:48] bits are not used for virtual address + // translation and they have to be set according to value of bit 47 in order + // get canonical address. +#ifdef DEBUG + // Check we have the result from a li macro-instruction. + Instr instr0 = instr_at(pc); + Instr instr1 = instr_at(pc + kInstrSize); + Instr instr2 = instr_at(pc + kInstrSize * 2); + DCHECK(IsLu12i_w(instr0) && IsOri(instr1) && IsLu32i_d(instr2) || + IsB(instr0)); +#endif + + Instr instr = instr_at(pc); + uint32_t* p = reinterpret_cast<uint32_t*>(pc); + if (IsB(instr)) { + int32_t offset = (target - pc) >> 2; + CHECK(is_int26(offset)); + offset = + ((offset & kImm16Mask) << kRkShift) | ((offset & kImm26Mask) >> 16); + *p = (instr & ~kImm26Mask) | offset; + if (icache_flush_mode != SKIP_ICACHE_FLUSH) { + FlushInstructionCache(pc, kInstrSize); + } + return; + } + uint32_t rd_code = GetRd(instr); + + // Must use 3 instructions to insure patchable code. + // lu12i_w rd, middle-20. + // ori rd, rd, low-12. + // lu32i_d rd, high-20. + *p = LU12I_W | (((target >> 12) & 0xfffff) << kRjShift) | rd_code; + *(p + 1) = + ORI | (target & 0xfff) << kRkShift | (rd_code << kRjShift) | rd_code; + *(p + 2) = LU32I_D | (((target >> 32) & 0xfffff) << kRjShift) | rd_code; + + if (icache_flush_mode != SKIP_ICACHE_FLUSH) { + FlushInstructionCache(pc, 3 * kInstrSize); + } +} + +UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler) + : available_(assembler->GetScratchRegisterList()), + old_available_(*available_) {} + +UseScratchRegisterScope::~UseScratchRegisterScope() { + *available_ = old_available_; +} + +Register UseScratchRegisterScope::Acquire() { + DCHECK_NOT_NULL(available_); + DCHECK_NE(*available_, 0); + int index = static_cast<int>(base::bits::CountTrailingZeros32(*available_)); + *available_ &= ~(1UL << index); + + return Register::from_code(index); +} + +bool UseScratchRegisterScope::hasAvailable() const { return *available_ != 0; } + +} // namespace internal +} // namespace v8 + +#endif // V8_TARGET_ARCH_LOONG64 |