diff options
Diffstat (limited to 'erts/emulator/beam/jit/arm/instr_map.cpp')
| -rw-r--r-- | erts/emulator/beam/jit/arm/instr_map.cpp | 90 |
1 files changed, 58 insertions, 32 deletions
diff --git a/erts/emulator/beam/jit/arm/instr_map.cpp b/erts/emulator/beam/jit/arm/instr_map.cpp index 651461112d..8d7ad6f45f 100644 --- a/erts/emulator/beam/jit/arm/instr_map.cpp +++ b/erts/emulator/beam/jit/arm/instr_map.cpp @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2020-2022. All Rights Reserved. + * Copyright Ericsson AB 2020-2023. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ using namespace asmjit; extern "C" { #include "erl_map.h" +#include "erl_term_hashing.h" #include "beam_common.h" } @@ -52,6 +53,11 @@ void BeamGlobalAssembler::emit_internal_hash_helper() { a.add(lower, lower, constant); a.add(upper, upper, constant); +#if defined(ERL_INTERNAL_HASH_CRC32C) + a.crc32cw(lower, hash, lower); + a.add(hash, hash, lower); + a.crc32cw(hash, hash, upper); +#else using rounds = std::initializer_list<std::tuple<a64::Gp, a64::Gp, a64::Gp, int>>; for (const auto &round : rounds{{lower, upper, hash, 13}, @@ -74,6 +80,7 @@ void BeamGlobalAssembler::emit_internal_hash_helper() { a.eor(r_a, r_a, r_c, arm::lsl(-shift)); } } +#endif #ifdef DBG_HASHMAP_COLLISION_BONANZA emit_enter_runtime_frame(); @@ -245,14 +252,14 @@ void BeamGlobalAssembler::emit_flatmap_get_element() { void BeamGlobalAssembler::emit_new_map_shared() { emit_enter_runtime_frame(); - emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs | + emit_enter_runtime<Update::eHeapAlloc | Update::eXRegs | Update::eReductions>(); a.mov(ARG1, c_p); load_x_reg_array(ARG2); runtime_call<5>(erts_gc_new_map); - emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs | + emit_leave_runtime<Update::eHeapAlloc | Update::eXRegs | Update::eReductions>(); emit_leave_runtime_frame(); @@ -272,22 +279,6 @@ void BeamModuleAssembler::emit_new_map(const ArgRegister &Dst, mov_arg(Dst, ARG1); } -void BeamGlobalAssembler::emit_i_new_small_map_lit_shared() { - emit_enter_runtime_frame(); - emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs | - Update::eReductions>(); - - a.mov(ARG1, c_p); - load_x_reg_array(ARG2); - runtime_call<5>(erts_gc_new_small_map_lit); - - emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs | - Update::eReductions>(); - emit_leave_runtime_frame(); - - a.ret(a64::x30); -} - void BeamModuleAssembler::emit_i_new_small_map_lit(const ArgRegister &Dst, const ArgWord &Live, const ArgLiteral &Keys, @@ -295,15 +286,50 @@ void BeamModuleAssembler::emit_i_new_small_map_lit(const ArgRegister &Dst, const Span<ArgVal> &args) { ASSERT(Size.get() == args.size()); - embed_vararg_rodata(args, ARG5); + emit_gc_test(ArgWord(0), + ArgWord(args.size() + MAP_HEADER_FLATMAP_SZ + 1), + Live); - ASSERT(Keys.isLiteral()); - mov_arg(ARG3, Keys); - mov_arg(ARG4, Live); + std::vector<ArgVal> data; + data.reserve(args.size() + MAP_HEADER_FLATMAP_SZ + 1); + data.push_back(ArgWord(MAP_HEADER_FLATMAP)); + data.push_back(Size); + data.push_back(Keys); - fragment_call(ga->get_i_new_small_map_lit_shared()); + bool dst_is_src = false; + for (auto arg : args) { + data.push_back(arg); + dst_is_src |= (arg == Dst); + } - mov_arg(Dst, ARG1); + if (dst_is_src) { + a.add(TMP1, HTOP, TAG_PRIMARY_BOXED); + } else { + auto ptr = init_destination(Dst, TMP1); + a.add(ptr.reg, HTOP, TAG_PRIMARY_BOXED); + flush_var(ptr); + } + + size_t size = data.size(); + unsigned i; + for (i = 0; i < size - 1; i += 2) { + if ((i % 128) == 0) { + check_pending_stubs(); + } + + auto [first, second] = load_sources(data[i], TMP2, data[i + 1], TMP3); + a.stp(first.reg, second.reg, arm::Mem(HTOP).post(sizeof(Eterm[2]))); + } + + if (i < size) { + mov_arg(arm::Mem(HTOP).post(sizeof(Eterm)), data[i]); + } + + if (dst_is_src) { + auto ptr = init_destination(Dst, TMP1); + mov_var(ptr, TMP1); + flush_var(ptr); + } } /* ARG1 = map @@ -369,7 +395,7 @@ void BeamModuleAssembler::emit_i_get_map_element(const ArgLabel &Fail, mov_arg(ARG1, Src); mov_arg(ARG2, Key); - if (masked_types(Key, BEAM_TYPE_MASK_IMMEDIATE) != BEAM_TYPE_NONE) { + if (maybe_one_of<BeamTypeId::MaybeImmediate>(Key)) { fragment_call(ga->get_i_get_map_element_shared()); a.b_ne(resolve_beam_label(Fail, disp1MB)); } else { @@ -540,14 +566,14 @@ void BeamModuleAssembler::emit_i_get_map_element_hash(const ArgLabel &Fail, /* ARG3 = live registers, ARG4 = update vector size, ARG5 = update vector. */ void BeamGlobalAssembler::emit_update_map_assoc_shared() { emit_enter_runtime_frame(); - emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs | + emit_enter_runtime<Update::eHeapAlloc | Update::eXRegs | Update::eReductions>(); a.mov(ARG1, c_p); load_x_reg_array(ARG2); runtime_call<5>(erts_gc_update_map_assoc); - emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs | + emit_leave_runtime<Update::eHeapAlloc | Update::eXRegs | Update::eReductions>(); emit_leave_runtime_frame(); @@ -579,14 +605,14 @@ void BeamModuleAssembler::emit_update_map_assoc(const ArgSource &Src, * Result is returned in RET, error is indicated by ZF. */ void BeamGlobalAssembler::emit_update_map_exact_guard_shared() { emit_enter_runtime_frame(); - emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs | + emit_enter_runtime<Update::eHeapAlloc | Update::eXRegs | Update::eReductions>(); a.mov(ARG1, c_p); load_x_reg_array(ARG2); runtime_call<5>(erts_gc_update_map_exact); - emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs | + emit_leave_runtime<Update::eHeapAlloc | Update::eXRegs | Update::eReductions>(); emit_leave_runtime_frame(); @@ -600,14 +626,14 @@ void BeamGlobalAssembler::emit_update_map_exact_body_shared() { Label error = a.newLabel(); emit_enter_runtime_frame(); - emit_enter_runtime<Update::eStack | Update::eHeap | Update::eXRegs | + emit_enter_runtime<Update::eHeapAlloc | Update::eXRegs | Update::eReductions>(); a.mov(ARG1, c_p); load_x_reg_array(ARG2); runtime_call<5>(erts_gc_update_map_exact); - emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs | + emit_leave_runtime<Update::eHeapAlloc | Update::eXRegs | Update::eReductions>(); emit_leave_runtime_frame(); |