diff options
Diffstat (limited to 'erts/emulator/beam/jit/beam_jit_main.cpp')
| -rw-r--r-- | erts/emulator/beam/jit/beam_jit_main.cpp | 142 |
1 files changed, 94 insertions, 48 deletions
diff --git a/erts/emulator/beam/jit/beam_jit_main.cpp b/erts/emulator/beam/jit/beam_jit_main.cpp index ef0ad6943a..0cd732039f 100644 --- a/erts/emulator/beam/jit/beam_jit_main.cpp +++ b/erts/emulator/beam/jit/beam_jit_main.cpp @@ -43,6 +43,8 @@ ErtsFrameLayout ERTS_WRITE_UNLIKELY(erts_frame_layout); #ifdef HAVE_LINUX_PERF_SUPPORT enum beamasm_perf_flags erts_jit_perf_support; #endif +/* Force use of single-mapped RWX memory for JIT code */ +int erts_jit_single_map = 0; /* * Special Beam instructions. @@ -54,7 +56,8 @@ ErtsCodePtr beam_exit; ErtsCodePtr beam_export_trampoline; ErtsCodePtr beam_bif_export_trap; ErtsCodePtr beam_continue_exit; -ErtsCodePtr beam_save_calls; +ErtsCodePtr beam_save_calls_export; +ErtsCodePtr beam_save_calls_fun; ErtsCodePtr beam_unloaded_fun; /* NOTE These should be the only variables containing trace instructions. @@ -65,7 +68,7 @@ ErtsCodePtr beam_unloaded_fun; ErtsCodePtr beam_return_to_trace; /* OpCode(i_return_to_trace) */ ErtsCodePtr beam_return_trace; /* OpCode(i_return_trace) */ ErtsCodePtr beam_exception_trace; /* UGLY also OpCode(i_return_trace) */ -ErtsCodePtr beam_return_time_trace; /* OpCode(i_return_time_trace) */ +ErtsCodePtr beam_call_trace_return; /* OpCode(i_call_trace_return) */ static JitAllocator *jit_allocator; @@ -86,7 +89,7 @@ static void install_bifs(void) { int i; ASSERT(beam_export_trampoline != NULL); - ASSERT(beam_save_calls != NULL); + ASSERT(beam_save_calls_export != NULL); for (i = 0; i < BIF_SIZE; i++) { BifEntry *entry; @@ -117,42 +120,56 @@ static void install_bifs(void) { } } -static JitAllocator *create_allocator(JitAllocator::CreateParams *params) { +static auto create_allocator(const JitAllocator::CreateParams ¶ms) { void *test_ro, *test_rw; + bool single_mapped; Error err; - auto *allocator = new JitAllocator(params); + auto *allocator = new JitAllocator(¶ms); err = allocator->alloc(&test_ro, &test_rw, 1); + + if (err == ErrorCode::kErrorOk) { + /* We can get dual-mapped memory when asking for single-mapped memory + * if the latter is not possible: return whether that happened. */ + single_mapped = (test_ro == test_rw); + } + allocator->release(test_ro); if (err == ErrorCode::kErrorOk) { - return allocator; + return std::make_pair(allocator, single_mapped); } delete allocator; - return nullptr; + return std::make_pair((JitAllocator *)nullptr, false); } static JitAllocator *pick_allocator() { - JitAllocator::CreateParams single_params; - single_params.reset(); + JitAllocator::CreateParams params; + JitAllocator *allocator; + bool single_mapped; + +#if defined(VALGRIND) + erts_jit_single_map = 1; +#elif defined(__APPLE__) && defined(__aarch64__) + /* Allocating dual-mapped executable memory on this platform is horribly + * slow, and provides little security benefits over the MAP_JIT per-thread + * permission scheme. Force single-mapped memory. + * + * 64-bit x86 still uses dual-mapped memory as it lacks support for per- + * thread permissions and thus gets unprotected RWX pages with MAP_JIT. */ + erts_jit_single_map = 1; +#endif #if defined(HAVE_LINUX_PERF_SUPPORT) /* `perf` has a hard time showing symbols for dual-mapped memory, so we'll * use single-mapped memory when enabled. */ if (erts_jit_perf_support & BEAMASM_PERF_ENABLED) { - if (auto *alloc = create_allocator(&single_params)) { - return alloc; - } - - ERTS_INTERNAL_ERROR("jit: Failed to allocate executable+writable " - "memory. Either allow this or disable the " - "'+JPperf' option."); + erts_jit_single_map = 1; } #endif -#if !defined(VALGRIND) /* Default to dual-mapped memory with separate executable and writable * regions of the same code. This is required for platforms that enforce * W^X, and we prefer it when available to catch errors sooner. @@ -162,28 +179,34 @@ static JitAllocator *pick_allocator() { * file descriptor per block on most platforms. The block sizes do grow * over time, but we don't want to waste half a dozen fds just to get to * the shell on platforms that are very fd-constrained. */ - JitAllocator::CreateParams dual_params; + params.reset(); + params.blockSize = 32 << 20; - dual_params.reset(); - dual_params.options = JitAllocatorOptions::kUseDualMapping, - dual_params.blockSize = 4 << 20; + allocator = nullptr; + single_mapped = false; - if (auto *alloc = create_allocator(&dual_params)) { - return alloc; - } else if (auto *alloc = create_allocator(&single_params)) { - return alloc; + if (!erts_jit_single_map) { + params.options = JitAllocatorOptions::kUseDualMapping; + std::tie(allocator, single_mapped) = create_allocator(params); } - ERTS_INTERNAL_ERROR("jit: Cannot allocate executable memory. Use the " - "interpreter instead."); -#elif defined(VALGRIND) - if (auto *alloc = create_allocator(&single_params)) { - return alloc; + if (allocator == nullptr) { + params.options &= ~JitAllocatorOptions::kUseDualMapping; + std::tie(allocator, single_mapped) = create_allocator(params); } - ERTS_INTERNAL_ERROR("jit: the valgrind emulator requires the ability to " - "allocate executable+writable memory."); -#endif + if (erts_jit_single_map && !single_mapped) { + ERTS_INTERNAL_ERROR("jit: Failed to allocate executable+writable " + "memory. Either allow this or disable both the " + "'+JPperf' and '+JMsingle' options."); + } + + if (allocator == nullptr) { + ERTS_INTERNAL_ERROR("jit: Cannot allocate executable memory. Use the " + "interpreter instead."); + } + + return allocator; } void beamasm_init() { @@ -208,10 +231,10 @@ void beamasm_init() { 0, op_i_return_to_trace, &beam_return_to_trace}, - {am_return_time_trace, + {am_call_trace_return, 0, - op_i_return_time_trace, - &beam_return_time_trace}}; + op_i_call_trace_return, + &beam_call_trace_return}}; Eterm mod_name; ERTS_DECL_AM(erts_beamasm); @@ -325,7 +348,8 @@ void beamasm_init() { /* These instructions rely on register contents, and can only be reached * from a `call_ext_*`-instruction or trapping from the emulator, hence the * lack of wrapper functions. */ - beam_save_calls = (ErtsCodePtr)bga->get_dispatch_save_calls(); + beam_save_calls_export = (ErtsCodePtr)bga->get_dispatch_save_calls_export(); + beam_save_calls_fun = (ErtsCodePtr)bga->get_dispatch_save_calls_fun(); beam_export_trampoline = (ErtsCodePtr)bga->get_export_trampoline(); /* Used when trappping to Erlang code from the emulator, setting up @@ -340,7 +364,7 @@ void beamasm_init() { beamasm_metadata_late_init(); } -bool BeamAssembler::hasCpuFeature(uint32_t featureId) { +bool BeamAssemblerCommon::hasCpuFeature(uint32_t featureId) { return cpuinfo.hasFeature(featureId); } @@ -365,6 +389,26 @@ extern "C" #endif } + void beamasm_unseal_module(const void *executable_region, + void *writable_region, + size_t size) { + (void)executable_region; + (void)writable_region; + (void)size; + + VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadWrite); + } + + void beamasm_seal_module(const void *executable_region, + void *writable_region, + size_t size) { + (void)executable_region; + (void)writable_region; + (void)size; + + VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadExecute); + } + void beamasm_flush_icache(const void *address, size_t size) { #ifdef DEBUG erts_debug_require_code_barrier(); @@ -389,7 +433,7 @@ extern "C" ETHR_COMPILER_BARRIER; - for (size_t i = start & ~ERTS_CACHE_LINE_MASK; i < end; + for (UWord i = start & ~ERTS_CACHE_LINE_MASK; i < end; i += ERTS_CACHE_LINE_SIZE) { __asm__ __volatile__("dc cvau, %0\n" "ic ivau, %0\n" ::"r"(i) @@ -473,9 +517,11 @@ extern "C" delete ba; } - void beamasm_purge_module(const void *native_module_exec, - void *native_module_rw) { - jit_allocator->release(const_cast<void *>(native_module_exec)); + void beamasm_purge_module(const void *executable_region, + void *writable_region, + size_t size) { + (void)size; + jit_allocator->release(const_cast<void *>(executable_region)); } ErtsCodePtr beamasm_get_code(void *instance, int label) { @@ -511,16 +557,16 @@ extern "C" } void beamasm_codegen(void *instance, - const void **native_module_exec, - void **native_module_rw, + const void **executable_region, + void **writable_region, const BeamCodeHeader *in_hdr, const BeamCodeHeader **out_exec_hdr, BeamCodeHeader **out_rw_hdr) { BeamModuleAssembler *ba = static_cast<BeamModuleAssembler *>(instance); ba->codegen(jit_allocator, - native_module_exec, - native_module_rw, + executable_region, + writable_region, in_hdr, out_exec_hdr, out_rw_hdr); @@ -562,7 +608,7 @@ extern "C" void beamasm_patch_import(void *instance, char *rw_base, int index, - BeamInstr import) { + const Export *import) { BeamModuleAssembler *ba = static_cast<BeamModuleAssembler *>(instance); ba->patchImport(rw_base, index, import); } @@ -578,7 +624,7 @@ extern "C" void beamasm_patch_lambda(void *instance, char *rw_base, int index, - BeamInstr fe) { + const ErlFunEntry *fe) { BeamModuleAssembler *ba = static_cast<BeamModuleAssembler *>(instance); ba->patchLambda(rw_base, index, fe); } |