diff options
Diffstat (limited to 'erts/emulator/beam/jit/x86/instr_guard_bifs.cpp')
| -rw-r--r-- | erts/emulator/beam/jit/x86/instr_guard_bifs.cpp | 759 |
1 files changed, 718 insertions, 41 deletions
diff --git a/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp index 9ef5486568..de243a45e6 100644 --- a/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp +++ b/erts/emulator/beam/jit/x86/instr_guard_bifs.cpp @@ -19,6 +19,7 @@ */ #include <algorithm> +#include <numeric> #include "beam_asm.hpp" extern "C" @@ -28,56 +29,285 @@ extern "C" #include "beam_catches.h" #include "beam_common.h" #include "code_ix.h" +#include "erl_map.h" } using namespace asmjit; -/* - * We considered specializing tuple_size/1, but ultimately didn't - * consider it worth doing. - * - * At the time of writing, there were 294 uses of tuple_size/1 - * in the OTP source code. (11 of them were in dialyzer.) - * - * The code size for the specialization was 34 bytes, - * while the code size for the bif1 instruction was 24 bytes. +/* ================================================================ + * '=:='/2 + * '=/='/2 + * '>='/2 + * '<'/2 + * ================================================================ */ -void BeamGlobalAssembler::emit_handle_hd_error() { - static ErtsCodeMFA mfa = {am_erlang, am_hd, 1}; +void BeamModuleAssembler::emit_bif_is_eq_ne_exact(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst, + Eterm fail_value, + Eterm succ_value) { + /* `mov_imm` may clobber the flags if either value is zero. */ + ASSERT(fail_value && succ_value); - a.mov(getXRef(0), RET); - a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADARG)); - a.mov(ARG4, imm(&mfa)); - a.jmp(labels[raise_exception]); + cmp_arg(getArgRef(LHS), RHS); + mov_imm(RET, succ_value); + + if (always_immediate(LHS) || always_immediate(RHS)) { + if (!LHS.isImmed() && !RHS.isImmed()) { + comment("simplified check since one argument is an immediate"); + } + mov_imm(ARG1, fail_value); + a.cmovne(RET, ARG1); + } else { + Label next = a.newLabel(); + + a.je(next); + + mov_arg(ARG1, LHS); + mov_arg(ARG2, RHS); + + emit_enter_runtime(); + runtime_call<2>(eq); + emit_leave_runtime(); + + a.test(RET, RET); + + mov_imm(RET, succ_value); + mov_imm(ARG1, fail_value); + a.cmove(RET, ARG1); + + a.bind(next); + } + + mov_arg(Dst, RET); } -/* - * At the time of implementation, there were 3285 uses of hd/1 in - * the OTP source code. Most of them were in code generated by - * yecc. - * - * The code size for this specialization of hd/1 is 21 bytes, - * while the code size for the bif1 instruction is 24 bytes. - */ -void BeamModuleAssembler::emit_bif_hd(const ArgSource &Src, - const ArgRegister &Hd) { - Label good_cons = a.newLabel(); +void BeamModuleAssembler::emit_bif_is_eq_exact(const ArgRegister &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + emit_bif_is_eq_ne_exact(LHS, RHS, Dst, am_false, am_true); +} - mov_arg(RET, Src); - a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); +void BeamModuleAssembler::emit_bif_is_ne_exact(const ArgRegister &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + emit_bif_is_eq_ne_exact(LHS, RHS, Dst, am_true, am_false); +} - a.short_().je(good_cons); - safe_fragment_call(ga->get_handle_hd_error()); +void BeamModuleAssembler::emit_cond_to_bool(uint32_t instId, + const ArgRegister &Dst) { + mov_imm(RET, am_true); + mov_imm(ARG1, am_false); + a.emit(instId, RET, ARG1); + mov_arg(Dst, RET); +} - a.bind(good_cons); +void BeamModuleAssembler::emit_bif_is_ge_lt(uint32_t instId, + const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + Label generic = a.newLabel(), make_boolean = a.newLabel(); + + mov_arg(ARG2, RHS); /* May clobber ARG1 */ + mov_arg(ARG1, LHS); + + if (always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>(LHS) && + always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>(RHS)) { + /* The only possible kind of immediate is a small and all other + * values are boxed, so we can test for smalls by testing boxed. */ + comment("simplified small test since all other types are boxed"); + if (always_small(LHS)) { + emit_is_not_boxed(generic, ARG2, dShort); + } else if (always_small(RHS)) { + emit_is_not_boxed(generic, ARG1, dShort); + } else { + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + emit_is_not_boxed(generic, RET, dShort); + } + } else { + /* Relative comparisons are overwhelmingly likely to be used on + * smalls, so we'll specialize those and keep the rest in a shared + * fragment. */ + if (always_small(RHS)) { + a.mov(RETd, ARG1d); + } else if (always_small(LHS)) { + a.mov(RETd, ARG2d); + } else { + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + } + + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(generic); + } + + /* Both arguments are smalls. */ + a.cmp(ARG1, ARG2); + a.short_().jmp(make_boolean); + + a.bind(generic); { - x86::Gp boxed_ptr = emit_ptr_val(RET, RET); - a.mov(ARG2, getCARRef(boxed_ptr)); - mov_arg(Hd, ARG2); + a.cmp(ARG1, ARG2); + a.short_().je(make_boolean); + safe_fragment_call(ga->get_arith_compare_shared()); } + + a.bind(make_boolean); + emit_cond_to_bool(instId, Dst); } +void BeamModuleAssembler::emit_bif_is_ge(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + bool both_small = always_small(LHS) && always_small(RHS); + + if (both_small && LHS.isRegister() && RHS.isImmed() && + Support::isInt32(RHS.as<ArgImmed>().get())) { + comment("simplified compare because one operand is an immediate small"); + a.cmp(getArgRef(LHS.as<ArgRegister>()), imm(RHS.as<ArgImmed>().get())); + emit_cond_to_bool(x86::Inst::kIdCmovl, Dst); + + return; + } else if (both_small && RHS.isRegister() && LHS.isImmed() && + Support::isInt32(LHS.as<ArgImmed>().get())) { + comment("simplified compare because one operand is an immediate small"); + a.cmp(getArgRef(RHS.as<ArgRegister>()), imm(LHS.as<ArgImmed>().get())); + emit_cond_to_bool(x86::Inst::kIdCmovg, Dst); + + return; + } + + emit_bif_is_ge_lt(x86::Inst::kIdCmovl, LHS, RHS, Dst); +} + +void BeamModuleAssembler::emit_bif_is_lt(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + emit_bif_is_ge_lt(x86::Inst::kIdCmovge, LHS, RHS, Dst); +} + +/* ================================================================ + * bit_size/1 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_bit_size(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgSource &Src, + const ArgRegister &Dst) { + if (!exact_type<BeamTypeId::Bitstring>(Src)) { + /* Unknown type. Use the standard BIF instruction. */ + emit_i_bif1(Src, Fail, Bif, Dst); + return; + } + + mov_arg(ARG2, Src); + + auto unit = getSizeUnit(Src); + bool is_bitstring = unit == 0 || std::gcd(unit, 8) != 8; + x86::Gp boxed_ptr = emit_ptr_val(ARG2, ARG2); + + if (is_bitstring) { + comment("inlined bit_size/1 because " + "its argument is a bitstring"); + } else { + comment("inlined and simplified bit_size/1 because " + "its argument is a binary"); + } + + if (is_bitstring) { + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + } + + a.mov(ARG1, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.shl(ARG1, imm(3 + _TAG_IMMED1_SIZE)); + + if (is_bitstring) { + Label not_sub_bin = a.newLabel(); + const auto diff_mask = _TAG_HEADER_SUB_BIN - _TAG_HEADER_REFC_BIN; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & diff_mask) != 0 && + (_TAG_HEADER_REFC_BIN & diff_mask) == 0 && + (_TAG_HEADER_HEAP_BIN & diff_mask) == 0); + a.test(RETb, imm(diff_mask)); + a.short_().jz(not_sub_bin); + + a.mov(RETb, emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize), 1)); + a.shl(RETb, imm(_TAG_IMMED1_SIZE)); + a.add(ARG1.r8(), RETb); + + a.bind(not_sub_bin); + } + + a.or_(ARG1, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, ARG1); +} + +/* ================================================================ + * byte_size/1 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_byte_size(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgSource &Src, + const ArgRegister &Dst) { + if (!exact_type<BeamTypeId::Bitstring>(Src)) { + /* Unknown type. Use the standard BIF instruction. */ + emit_i_bif1(Src, Fail, Bif, Dst); + return; + } + + mov_arg(ARG2, Src); + + auto unit = getSizeUnit(Src); + bool is_bitstring = unit == 0 || std::gcd(unit, 8) != 8; + x86::Gp boxed_ptr = emit_ptr_val(ARG2, ARG2); + + if (is_bitstring) { + comment("inlined byte_size/1 because " + "its argument is a bitstring"); + } else { + comment("inlined and simplified byte_size/1 because " + "its argument is a binary"); + } + + if (is_bitstring) { + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + } + + a.mov(ARG1, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + + if (is_bitstring) { + Label not_sub_bin = a.newLabel(); + const auto diff_mask = _TAG_HEADER_SUB_BIN - _TAG_HEADER_REFC_BIN; + ERTS_CT_ASSERT((_TAG_HEADER_SUB_BIN & diff_mask) != 0 && + (_TAG_HEADER_REFC_BIN & diff_mask) == 0 && + (_TAG_HEADER_HEAP_BIN & diff_mask) == 0); + a.test(RETb, imm(diff_mask)); + a.short_().jz(not_sub_bin); + + a.mov(RETb, emit_boxed_val(boxed_ptr, offsetof(ErlSubBin, bitsize), 1)); + a.test(RETb, RETb); + a.setne(RETb); + a.movzx(RETd, RETb); + a.add(ARG1, RET); + + a.bind(not_sub_bin); + } + + a.shl(ARG1, imm(_TAG_IMMED1_SIZE)); + a.or_(ARG1, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, ARG1); +} + +/* ================================================================ + * element/2 + * ================================================================ + */ + void BeamGlobalAssembler::emit_handle_element_error() { static ErtsCodeMFA mfa = {am_erlang, am_element, 2}; @@ -161,15 +391,14 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, /* * Try to optimize the use of a tuple as a lookup table. */ - if (exact_type(Pos, BEAM_TYPE_INTEGER) && Tuple.isLiteral()) { + if (exact_type<BeamTypeId::Integer>(Pos) && Tuple.isLiteral()) { Eterm tuple = beamfile_get_literal(beam, Tuple.as<ArgLiteral>().get()); if (is_tuple(tuple)) { Label error = a.newLabel(), next = a.newLabel(); Sint size = Sint(arityval(*tuple_val(tuple))); - auto [min, max] = getIntRange(Pos); - bool is_bounded = min <= max; - bool can_fail = !is_bounded || min < 1 || size < max; + auto [min, max] = getClampedRange(Pos); + bool can_fail = min < 1 || size < max; comment("skipped tuple test since source is always a literal " "tuple"); @@ -189,13 +418,13 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, a.mov(RET, ARG1); a.sar(RET, imm(_TAG_IMMED1_SIZE)); - if (is_bounded && min >= 1) { + if (min >= 1) { comment("skipped check for position =:= 0 since it is always " ">= 1"); } else { a.short_().jz(error); } - if (is_bounded && min >= 0 && size >= max) { + if (min >= 0 && size >= max) { comment("skipped check for negative position and position " "beyond tuple"); } else { @@ -236,7 +465,7 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, x86::Gp boxed_ptr = emit_ptr_val(ARG3, ARG2); - if (exact_type(Tuple, BEAM_TYPE_TUPLE)) { + if (exact_type<BeamTypeId::Tuple>(Tuple)) { comment("skipped tuple test since source is always a tuple"); ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); a.cmp(emit_boxed_val(boxed_ptr, 0, sizeof(Uint32)), @@ -317,3 +546,451 @@ void BeamModuleAssembler::emit_bif_element(const ArgLabel &Fail, mov_arg(Dst, RET); } + +/* ================================================================ + * hd/1 + * ================================================================ + */ + +void BeamGlobalAssembler::emit_handle_hd_error() { + static ErtsCodeMFA mfa = {am_erlang, am_hd, 1}; + + a.mov(getXRef(0), RET); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADARG)); + a.mov(ARG4, imm(&mfa)); + a.jmp(labels[raise_exception]); +} + +/* + * At the time of implementation, there were 3285 uses of hd/1 in + * the OTP source code. Most of them were in code generated by + * yecc. + * + * The code size for this specialization of hd/1 is 21 bytes, + * while the code size for the bif1 instruction is 24 bytes. + */ + +void BeamModuleAssembler::emit_bif_hd(const ArgSource &Src, + const ArgRegister &Hd) { + Label good_cons = a.newLabel(); + + mov_arg(RET, Src); + a.test(RETb, imm(_TAG_PRIMARY_MASK - TAG_PRIMARY_LIST)); + + a.short_().je(good_cons); + safe_fragment_call(ga->get_handle_hd_error()); + + a.bind(good_cons); + { + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); + a.mov(ARG2, getCARRef(boxed_ptr)); + mov_arg(Hd, ARG2); + } +} + +/* ================================================================ + * is_map_key/2 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_is_map_key(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgSource &Key, + const ArgSource &Src, + const ArgRegister &Dst) { + if (!exact_type<BeamTypeId::Map>(Src)) { + emit_i_bif2(Key, Src, Fail, Bif, Dst); + return; + } + + comment("inlined BIF is_map_key/2"); + + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + + if (maybe_one_of<BeamTypeId::MaybeImmediate>(Key) && + hasCpuFeature(CpuFeatures::X86::kBMI2)) { + safe_fragment_call(ga->get_i_get_map_element_shared()); + emit_cond_to_bool(x86::Inst::kIdCmovne, Dst); + } else { + emit_enter_runtime(); + runtime_call<2>(get_map_element); + emit_leave_runtime(); + + emit_test_the_non_value(RET); + emit_cond_to_bool(x86::Inst::kIdCmove, Dst); + } +} + +/* ================================================================ + * map_get/2 + * ================================================================ + */ + +void BeamGlobalAssembler::emit_handle_map_get_badmap() { + static ErtsCodeMFA mfa = {am_erlang, am_map_get, 2}; + a.mov(getXRef(0), ARG2); + a.mov(getXRef(1), ARG1); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADMAP)); + a.mov(x86::qword_ptr(c_p, offsetof(Process, fvalue)), ARG1); + a.mov(ARG4, imm(&mfa)); + a.jmp(labels[raise_exception]); +} + +void BeamGlobalAssembler::emit_handle_map_get_badkey() { + static ErtsCodeMFA mfa = {am_erlang, am_map_get, 2}; + a.mov(getXRef(0), ARG2); + a.mov(getXRef(1), ARG1); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADKEY)); + a.mov(x86::qword_ptr(c_p, offsetof(Process, fvalue)), ARG2); + a.mov(ARG4, imm(&mfa)); + a.jmp(labels[raise_exception]); +} + +void BeamModuleAssembler::emit_bif_map_get(const ArgLabel &Fail, + const ArgSource &Key, + const ArgSource &Src, + const ArgRegister &Dst) { + Label good_key = a.newLabel(); + + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + + if (exact_type<BeamTypeId::Map>(Src)) { + comment("skipped test for map for known map argument"); + } else { + Label bad_map = a.newLabel(); + Label good_map = a.newLabel(); + + if (Fail.get() == 0) { + emit_is_boxed(bad_map, Src, ARG1); + } else { + emit_is_boxed(resolve_beam_label(Fail), Src, ARG1); + } + + /* As an optimization for the `error | #{}` case, skip checking the + * header word when we know that the only possible boxed type + * is a map. */ + if (masked_types<BeamTypeId::MaybeBoxed>(Src) == BeamTypeId::Map) { + comment("skipped header test since we know it's a map when boxed"); + if (Fail.get() == 0) { + a.short_().jmp(good_map); + } + } else { + x86::Gp boxed_ptr = emit_ptr_val(RET, ARG1); + a.mov(RET, emit_boxed_val(boxed_ptr)); + a.and_(RETb, imm(_TAG_HEADER_MASK)); + a.cmp(RETb, imm(_TAG_HEADER_MAP)); + if (Fail.get() == 0) { + a.short_().je(good_map); + } else { + a.jne(resolve_beam_label(Fail)); + } + } + + a.bind(bad_map); + if (Fail.get() == 0) { + fragment_call(ga->get_handle_map_get_badmap()); + } + + a.bind(good_map); + } + + if (maybe_one_of<BeamTypeId::MaybeImmediate>(Key) && + hasCpuFeature(CpuFeatures::X86::kBMI2)) { + safe_fragment_call(ga->get_i_get_map_element_shared()); + if (Fail.get() == 0) { + a.je(good_key); + } else { + a.jne(resolve_beam_label(Fail)); + } + } else { + emit_enter_runtime(); + runtime_call<2>(get_map_element); + emit_leave_runtime(); + + emit_test_the_non_value(RET); + if (Fail.get() == 0) { + a.short_().jne(good_key); + } else { + a.je(resolve_beam_label(Fail)); + } + } + + if (Fail.get() == 0) { + mov_arg(ARG1, Src); + mov_arg(ARG2, Key); + fragment_call(ga->get_handle_map_get_badkey()); + } + + a.bind(good_key); + mov_arg(Dst, RET); +} + +/* ================================================================ + * map_size/1 + * ================================================================ + */ + +void BeamGlobalAssembler::emit_handle_map_size_error() { + static ErtsCodeMFA mfa = {am_erlang, am_map_size, 1}; + + a.mov(getXRef(0), RET); + a.mov(x86::qword_ptr(c_p, offsetof(Process, fvalue)), RET); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADMAP)); + a.mov(ARG4, imm(&mfa)); + a.jmp(labels[raise_exception]); +} + +void BeamModuleAssembler::emit_bif_map_size(const ArgLabel &Fail, + const ArgSource &Src, + const ArgRegister &Dst) { + Label error = a.newLabel(), good_map = a.newLabel(); + + mov_arg(RET, Src); + + if (Fail.get() == 0) { + emit_is_boxed(error, Src, RET); + } else { + emit_is_boxed(resolve_beam_label(Fail), Src, RET); + } + + x86::Gp boxed_ptr = emit_ptr_val(x86::rdx, RET); + + if (exact_type<BeamTypeId::Map>(Src)) { + comment("skipped type check because the argument is always a map"); + a.bind(error); /* Never referenced. */ + } else { + a.mov(x86::ecx, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(x86::cl, imm(_TAG_HEADER_MASK)); + a.cmp(x86::cl, imm(_TAG_HEADER_MAP)); + if (Fail.get() == 0) { + a.short_().je(good_map); + + a.bind(error); + safe_fragment_call(ga->get_handle_map_size_error()); + } else { + a.jne(resolve_beam_label(Fail)); + a.bind(error); /* Never referenced. */ + } + } + + a.bind(good_map); + { + ERTS_CT_ASSERT(offsetof(flatmap_t, size) == sizeof(Eterm)); + a.mov(RET, emit_boxed_val(boxed_ptr, sizeof(Eterm))); + a.shl(RET, imm(4)); + a.or_(RETb, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, RET); + } +} + +/* ================================================================ + * min/2 + * max/2 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_min_max(uint32_t instId, + const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + Label generic = a.newLabel(), do_cmov = a.newLabel(); + bool both_small = always_small(LHS) && always_small(RHS); + bool need_generic = !both_small; + + mov_arg(ARG2, RHS); /* May clobber ARG1 */ + mov_arg(ARG1, LHS); + + if (both_small) { + comment("skipped test for small operands since they are always small"); + } else if (always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>( + LHS) && + always_small(RHS)) { + emit_is_not_boxed(generic, ARG1, dShort); + } else if (always_small(LHS) && + always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>( + RHS)) { + emit_is_not_boxed(generic, ARG2, dShort); + } else if (always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>( + LHS) && + always_one_of<BeamTypeId::Integer, BeamTypeId::AlwaysBoxed>( + RHS)) { + comment("simplified small test since all other types are boxed"); + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + emit_is_not_boxed(generic, RETb, dShort); + } else { + /* Relative comparisons are overwhelmingly likely to be used on + * smalls, so we'll specialize those and keep the rest in a shared + * fragment. */ + if (RHS.isSmall()) { + a.mov(RETd, ARG1d); + } else if (LHS.isSmall()) { + a.mov(RETd, ARG2d); + } else { + a.mov(RETd, ARG1d); + a.and_(RETd, ARG2d); + } + + a.and_(RETb, imm(_TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_SMALL)); + a.short_().jne(generic); + } + + /* Both arguments are smalls. */ + a.cmp(ARG1, ARG2); + if (need_generic) { + a.short_().jmp(do_cmov); + } + + a.bind(generic); + if (need_generic) { + a.cmp(ARG1, ARG2); + a.short_().je(do_cmov); + a.push(ARG1); + a.push(ARG2); + safe_fragment_call(ga->get_arith_compare_shared()); + a.pop(ARG2); + a.pop(ARG1); + } + + a.bind(do_cmov); + a.emit(instId, ARG1, ARG2); + mov_arg(Dst, ARG1); +} + +void BeamModuleAssembler::emit_bif_max(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + emit_bif_min_max(x86::Inst::kIdCmovl, LHS, RHS, Dst); +} + +void BeamModuleAssembler::emit_bif_min(const ArgSource &LHS, + const ArgSource &RHS, + const ArgRegister &Dst) { + emit_bif_min_max(x86::Inst::kIdCmovg, LHS, RHS, Dst); +} + +/* ================================================================ + * node/1 + * ================================================================ + */ + +void BeamGlobalAssembler::emit_handle_node_error() { + static ErtsCodeMFA mfa = {am_erlang, am_node, 1}; + a.mov(getXRef(0), ARG1); + a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), imm(BADARG)); + a.mov(ARG4, imm(&mfa)); + + a.jmp(labels[raise_exception]); +} + +void BeamModuleAssembler::emit_bif_node(const ArgLabel &Fail, + const ArgRegister &Src, + const ArgRegister &Dst) { + bool always_identifier = always_one_of<BeamTypeId::Identifier>(Src); + Label test_internal = a.newLabel(); + Label internal = a.newLabel(); + Label next = a.newLabel(); + Label fail; + + if (Fail.get() == 0 && !always_identifier) { + fail = a.newLabel(); + } + + mov_arg(ARG1, Src); + emit_is_boxed(test_internal, Src, ARG1); + + x86::Gp boxed_ptr = emit_ptr_val(ARG2, ARG1); + + if (!always_one_of<BeamTypeId::Pid, BeamTypeId::Port>(Src)) { + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + a.and_(RETd, imm(_TAG_HEADER_MASK)); + } + + if (maybe_one_of<BeamTypeId::Reference>(Src)) { + a.cmp(RETb, imm(_TAG_HEADER_REF)); + a.short_().je(internal); + } + + if (!always_identifier) { + Label external = a.newLabel(); + + ERTS_CT_ASSERT((_TAG_HEADER_EXTERNAL_PORT - _TAG_HEADER_EXTERNAL_PID) >> + _TAG_PRIMARY_SIZE == + 1); + ERTS_CT_ASSERT((_TAG_HEADER_EXTERNAL_REF - _TAG_HEADER_EXTERNAL_PORT) >> + _TAG_PRIMARY_SIZE == + 1); + a.sub(RETb, imm(_TAG_HEADER_EXTERNAL_PID)); + a.cmp(RETb, imm(_TAG_HEADER_EXTERNAL_REF - _TAG_HEADER_EXTERNAL_PID)); + if (Fail.get() == 0) { + a.short_().jbe(external); + + a.bind(fail); + fragment_call(ga->get_handle_node_error()); + } else { + a.ja(resolve_beam_label(Fail)); + } + + a.bind(external); + } + + a.mov(ARG1, emit_boxed_val(boxed_ptr, offsetof(ExternalThing, node))); + a.short_().jmp(next); + + a.bind(test_internal); + if (!always_identifier) { + /* Since pids and ports differ by a single bit, we can + * simplify the check by clearing said bit and comparing + * against the lesser one. */ + ERTS_CT_ASSERT(_TAG_IMMED1_PORT - _TAG_IMMED1_PID == 0x4); + a.mov(RETd, ARG1d); + a.and_(RETb, + imm(~(_TAG_IMMED1_PORT - _TAG_IMMED1_PID) & _TAG_IMMED1_MASK)); + a.cmp(RETb, imm(_TAG_IMMED1_PID)); + if (Fail.get() == 0) { + a.short_().jne(fail); + } else { + a.jne(resolve_beam_label(Fail)); + } + } + + a.bind(internal); + a.mov(ARG1, imm(&erts_this_node)); + a.mov(ARG1, x86::qword_ptr(ARG1)); + + a.bind(next); + a.mov(ARG1, x86::qword_ptr(ARG1, offsetof(ErlNode, sysname))); + mov_arg(Dst, ARG1); +} + +/* ================================================================ + * tuple_size/1 + * ================================================================ + */ + +void BeamModuleAssembler::emit_bif_tuple_size(const ArgWord &Bif, + const ArgLabel &Fail, + const ArgRegister &Src, + const ArgRegister &Dst) { + if (exact_type<BeamTypeId::Tuple>(Src)) { + comment("inlined tuple_size/1 because the argument is always a tuple"); + mov_arg(RET, Src); + + /* Instructions operating on dwords are shorter. */ + ERTS_CT_ASSERT(Support::isInt32(make_arityval(MAX_ARITYVAL))); + x86::Gp boxed_ptr = emit_ptr_val(RET, RET); + a.mov(RETd, emit_boxed_val(boxed_ptr, 0, sizeof(Uint32))); + + ERTS_CT_ASSERT(_HEADER_ARITY_OFFS - _TAG_IMMED1_SIZE > 0); + ERTS_CT_ASSERT(_TAG_IMMED1_SMALL == _TAG_IMMED1_MASK); + a.shr(RETd, imm(_HEADER_ARITY_OFFS - _TAG_IMMED1_SIZE)); + a.or_(RETb, imm(_TAG_IMMED1_SMALL)); + mov_arg(Dst, RET); + } else { + /* Unknown type. Use the standard BIF instruction. */ + emit_i_bif1(Src, Fail, Bif, Dst); + } +} |