diff options
| author | Alan Wu <XrXr@users.noreply.github.com> | 2022-08-26 14:02:51 -0400 |
|---|---|---|
| committer | Takashi Kokubun <takashikkbn@gmail.com> | 2022-08-29 09:09:41 -0700 |
| commit | 46007b88af82d6ff22fc01edb7c74922dfa5c68a (patch) | |
| tree | 342e6182f62573a1b6ee8b89017559cea7432dd3 | |
| parent | 29e0713a1272cb63f1e3cebfab85dec2424ead0f (diff) | |
| download | ruby-46007b88af82d6ff22fc01edb7c74922dfa5c68a.tar.gz | |
A64: Only clear icache when writing out new code (https://github.com/Shopify/ruby/pull/442)
Previously we cleared the cache for all the code in the system when we
flip memory protection, which was prohibitively expensive since the
operation is not constant time. Instead, only clear the cache for the
memory region of newly written code when we write out new code.
This brings the runtime for the 30k_if_else test down to about 6 seconds
from the previous 45 seconds on my laptop.
| -rw-r--r-- | yjit.c | 10 | ||||
| -rw-r--r-- | yjit/bindgen/src/main.rs | 1 | ||||
| -rw-r--r-- | yjit/src/backend/arm64/mod.rs | 10 | ||||
| -rw-r--r-- | yjit/src/cruby_bindings.inc.rs | 6 |
4 files changed, 25 insertions, 2 deletions
@@ -78,11 +78,17 @@ rb_yjit_mark_executable(void *mem_block, uint32_t mem_size) rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s\n", mem_block, (unsigned long)mem_size, strerror(errno)); } +} +// `start` is inclusive and `end` is exclusive. +void +rb_yjit_icache_invalidate(void *start, void *end) +{ // Clear/invalidate the instruction cache. Compiles to nothing on x86_64 - // but required on ARM. On Darwin it's the same as calling sys_icache_invalidate(). + // but required on ARM before running freshly written code. + // On Darwin it's the same as calling sys_icache_invalidate(). #ifdef __GNUC__ - __builtin___clear_cache(mem_block, (char *)mem_block + mem_size); + __builtin___clear_cache(start, end); #endif } diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index a4c0b98504..df4083638d 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -273,6 +273,7 @@ fn main() { .allowlist_function("rb_RSTRING_LEN") .allowlist_function("rb_ENCODING_GET") .allowlist_function("rb_yjit_exit_locations_dict") + .allowlist_function("rb_yjit_icache_invalidate") // from vm_sync.h .allowlist_function("rb_vm_barrier") diff --git a/yjit/src/backend/arm64/mod.rs b/yjit/src/backend/arm64/mod.rs index 08eb5efa3f..e63b011391 100644 --- a/yjit/src/backend/arm64/mod.rs +++ b/yjit/src/backend/arm64/mod.rs @@ -961,6 +961,16 @@ impl Assembler }; } + // Invalidate icache for newly written out region so we don't run + // stale code. + { + let start = cb.get_ptr(start_write_pos).raw_ptr(); + let write_ptr = cb.get_write_ptr().raw_ptr(); + let codeblock_end = cb.get_ptr(cb.get_mem_size()).raw_ptr(); + let end = std::cmp::min(write_ptr, codeblock_end); + unsafe { rb_yjit_icache_invalidate(start as _, end as _) }; + } + gc_offsets } diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 83e9f580bf..591408e1da 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1021,6 +1021,12 @@ extern "C" { pub fn rb_yjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); } extern "C" { + pub fn rb_yjit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); +} +extern "C" { pub fn rb_yjit_exit_locations_dict( yjit_raw_samples: *mut VALUE, yjit_line_samples: *mut ::std::os::raw::c_int, |