From 00daac3fbc798884028754a482999e8fc1ce63f9 Mon Sep 17 00:00:00 2001 From: zherczeg Date: Thu, 27 Aug 2020 06:19:17 +0000 Subject: JIT compiler update. git-svn-id: svn://vcs.exim.org/pcre2/code/trunk@1268 6239d852-aaf2-0410-a92c-79f79f948069 --- src/pcre2_jit_compile.c | 8 +- src/pcre2_jit_misc.c | 2 +- src/sljit/sljitConfig.h | 44 +++--- src/sljit/sljitConfigInternal.h | 66 ++++----- src/sljit/sljitExecAllocator.c | 59 ++++---- src/sljit/sljitLir.c | 28 ++-- src/sljit/sljitLir.h | 33 ++--- src/sljit/sljitNativeARM_32.c | 80 ++++++++++- src/sljit/sljitNativeARM_64.c | 32 ++--- src/sljit/sljitNativeARM_T2_32.c | 23 ++-- src/sljit/sljitNativeMIPS_32.c | 11 +- src/sljit/sljitNativeMIPS_64.c | 12 +- src/sljit/sljitNativeMIPS_common.c | 7 +- src/sljit/sljitNativePPC_32.c | 11 +- src/sljit/sljitNativePPC_64.c | 12 +- src/sljit/sljitNativePPC_common.c | 3 +- src/sljit/sljitNativeSPARC_32.c | 11 +- src/sljit/sljitNativeSPARC_common.c | 3 +- src/sljit/sljitNativeTILEGX_64.c | 4 +- src/sljit/sljitNativeX86_common.c | 55 ++++---- src/sljit/sljitProtExecAllocator.c | 139 +++++++++---------- src/sljit/sljitUtils.c | 261 +++++++++++++----------------------- src/sljit/sljitWXExecAllocator.c | 225 +++++++++++++++++++++++++++++++ 23 files changed, 690 insertions(+), 439 deletions(-) create mode 100644 src/sljit/sljitWXExecAllocator.c diff --git a/src/pcre2_jit_compile.c b/src/pcre2_jit_compile.c index 2bd4275..edf64d5 100644 --- a/src/pcre2_jit_compile.c +++ b/src/pcre2_jit_compile.c @@ -13659,7 +13659,7 @@ if (common->has_then) set_then_offsets(common, common->start, NULL); } -compiler = sljit_create_compiler(allocator_data); +compiler = sljit_create_compiler(allocator_data, NULL); if (!compiler) { SLJIT_FREE(common->optimized_cbracket, allocator_data); @@ -14061,7 +14061,7 @@ else { /* This case is highly unlikely since we just recently freed a lot of memory. Not impossible though. */ - sljit_free_code(executable_func); + sljit_free_code(executable_func, NULL); PRIV(jit_free_rodata)(common->read_only_data_head, allocator_data); return PCRE2_ERROR_NOMEMORY; } @@ -14175,13 +14175,13 @@ if (executable_allocator_is_working == 0) /* Checks whether the executable allocator is working. This check might run multiple times in multi-threaded environments, but the result should not be affected by it. */ - void *ptr = SLJIT_MALLOC_EXEC(32); + void *ptr = SLJIT_MALLOC_EXEC(32, NULL); executable_allocator_is_working = -1; if (ptr != NULL) { - SLJIT_FREE_EXEC(((sljit_u8*)(ptr)) + SLJIT_EXEC_OFFSET(ptr)); + SLJIT_FREE_EXEC(((sljit_u8*)(ptr)) + SLJIT_EXEC_OFFSET(ptr), NULL); executable_allocator_is_working = 1; } } diff --git a/src/pcre2_jit_misc.c b/src/pcre2_jit_misc.c index 36abdba..ec924e0 100644 --- a/src/pcre2_jit_misc.c +++ b/src/pcre2_jit_misc.c @@ -89,7 +89,7 @@ int i; for (i = 0; i < JIT_NUMBER_OF_COMPILE_MODES; i++) { if (functions->executable_funcs[i] != NULL) - sljit_free_code(functions->executable_funcs[i]); + sljit_free_code(functions->executable_funcs[i], NULL); PRIV(jit_free_rodata)(functions->read_only_data_heads[i], allocator_data); } diff --git a/src/sljit/sljitConfig.h b/src/sljit/sljitConfig.h index 4560450..522db89 100644 --- a/src/sljit/sljitConfig.h +++ b/src/sljit/sljitConfig.h @@ -24,19 +24,19 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _SLJIT_CONFIG_H_ -#define _SLJIT_CONFIG_H_ +#ifndef SLJIT_CONFIG_H_ +#define SLJIT_CONFIG_H_ #ifdef __cplusplus extern "C" { #endif -/* --------------------------------------------------------------------- */ -/* Custom defines */ -/* --------------------------------------------------------------------- */ - -/* Put your custom defines here. This empty section will never change - which helps maintaining patches (with diff / patch utilities). */ +/* + This file contains the basic configuration options for the SLJIT compiler + and their default values. These options can be overridden in the + sljitConfigPre.h header file when SLJIT_HAVE_CONFIG_PRE is set to a + non-zero value. +*/ /* --------------------------------------------------------------------- */ /* Architecture */ @@ -63,12 +63,6 @@ extern "C" { /* Utilities */ /* --------------------------------------------------------------------- */ -/* Useful for thread-safe compiling of global functions. */ -#ifndef SLJIT_UTIL_GLOBAL_LOCK -/* Enabled by default */ -#define SLJIT_UTIL_GLOBAL_LOCK 1 -#endif - /* Implements a stack like data structure (by using mmap / VirtualAlloc */ /* or a custom allocator). */ #ifndef SLJIT_UTIL_STACK @@ -108,15 +102,31 @@ extern "C" { /* When SLJIT_PROT_EXECUTABLE_ALLOCATOR is enabled SLJIT uses an allocator which does not set writable and executable - permission flags at the same time. The trade-of is increased - memory consumption and disabled dynamic code modifications. */ + permission flags at the same time. + Instead, it creates a shared memory segment (usually backed by a file) + and maps it twice, with different permissions, depending on the use + case. + The trade-off is increased use of virtual memory, incompatibility with + fork(), and some possible additional security risks by the use of + publicly accessible files for the generated code. */ #ifndef SLJIT_PROT_EXECUTABLE_ALLOCATOR /* Disabled by default. */ #define SLJIT_PROT_EXECUTABLE_ALLOCATOR 0 #endif +/* When SLJIT_WX_EXECUTABLE_ALLOCATOR is enabled SLJIT uses an + allocator which does not set writable and executable permission + flags at the same time. + Instead, it creates a new independent map on each invocation and + switches permissions at the underlying pages as needed. + The trade-off is increased memory use and degraded performance. */ +#ifndef SLJIT_WX_EXECUTABLE_ALLOCATOR +/* Disabled by default. */ +#define SLJIT_WX_EXECUTABLE_ALLOCATOR 0 #endif +#endif /* !SLJIT_EXECUTABLE_ALLOCATOR */ + /* Force cdecl calling convention even if a better calling convention (e.g. fastcall) is supported by the C compiler. If this option is disabled (this is the default), functions @@ -159,4 +169,4 @@ extern "C" { } /* extern "C" */ #endif -#endif +#endif /* SLJIT_CONFIG_H_ */ diff --git a/src/sljit/sljitConfigInternal.h b/src/sljit/sljitConfigInternal.h index 049ed2f..e75c404 100644 --- a/src/sljit/sljitConfigInternal.h +++ b/src/sljit/sljitConfigInternal.h @@ -24,8 +24,8 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _SLJIT_CONFIG_INTERNAL_H_ -#define _SLJIT_CONFIG_INTERNAL_H_ +#ifndef SLJIT_CONFIG_INTERNAL_H_ +#define SLJIT_CONFIG_INTERNAL_H_ #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \ || (defined SLJIT_DEBUG && SLJIT_DEBUG && (!defined(SLJIT_ASSERT) || !defined(SLJIT_UNREACHABLE))) @@ -81,7 +81,7 @@ extern "C" { Other macros: SLJIT_FUNC : calling convention attribute for both calling JIT from C and C calling back from JIT - SLJIT_W(number) : defining 64 bit constants on 64 bit architectures (compiler independent helper) + SLJIT_W(number) : defining 64 bit constants on 64 bit architectures (platform independent helper) */ /*****************/ @@ -210,18 +210,16 @@ extern "C" { /***********************************************************/ #ifdef SLJIT_CONFIG_X86 -#if defined(__CET__) + +#if defined(__CET__) && !(defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) #define SLJIT_CONFIG_X86_CET 1 #endif -#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) -#if defined(__GNUC__) -#if !defined (__SHSTK__) -#error "-mshstk is needed to compile with -fcf-protection" -#endif + +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined(__GNUC__) #include #endif -#endif -#endif + +#endif /* SLJIT_CONFIG_X86 */ /**********************************/ /* External function definitions. */ @@ -453,10 +451,14 @@ typedef double sljit_f64; #if (defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) #define SLJIT_W(w) (w##l) #elif (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE) +#ifdef _WIN64 #define SLJIT_W(w) (w##ll) -#else +#else /* !windows */ +#define SLJIT_W(w) (w##l) +#endif /* windows */ +#else /* 32 bit */ #define SLJIT_W(w) (w) -#endif +#endif /* unknown */ #endif /* !SLJIT_W */ @@ -549,17 +551,19 @@ typedef double sljit_f64; #ifndef SLJIT_FUNC -#if (defined SLJIT_USE_CDECL_CALLING_CONVENTION && SLJIT_USE_CDECL_CALLING_CONVENTION) +#if (defined SLJIT_USE_CDECL_CALLING_CONVENTION && SLJIT_USE_CDECL_CALLING_CONVENTION) \ + || !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) -/* Force cdecl. */ #define SLJIT_FUNC -#elif (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) - -#if defined(__GNUC__) && !defined(__APPLE__) +#elif defined(__GNUC__) && !defined(__APPLE__) +#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) #define SLJIT_FUNC __attribute__ ((fastcall)) #define SLJIT_X86_32_FASTCALL 1 +#else +#define SLJIT_FUNC +#endif /* gcc >= 3.4 */ #elif defined(_MSC_VER) @@ -573,16 +577,10 @@ typedef double sljit_f64; #else /* Unknown compiler. */ -/* The cdecl attribute is the default. */ +/* The cdecl calling convention is usually the x86 default. */ #define SLJIT_FUNC -#endif - -#else /* Non x86-32 architectures. */ - -#define SLJIT_FUNC - -#endif /* SLJIT_CONFIG_X86_32 */ +#endif /* SLJIT_USE_CDECL_CALLING_CONVENTION */ #endif /* !SLJIT_FUNC */ @@ -613,8 +611,16 @@ determine the next executed instruction after return. */ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size); SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr); SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void); -#define SLJIT_MALLOC_EXEC(size) sljit_malloc_exec(size) -#define SLJIT_FREE_EXEC(ptr) sljit_free_exec(ptr) +#define SLJIT_BUILTIN_MALLOC_EXEC(size, exec_allocator_data) sljit_malloc_exec(size) +#define SLJIT_BUILTIN_FREE_EXEC(ptr, exec_allocator_data) sljit_free_exec(ptr) + +#ifndef SLJIT_MALLOC_EXEC +#define SLJIT_MALLOC_EXEC(size, exec_allocator_data) SLJIT_BUILTIN_MALLOC_EXEC((size), (exec_allocator_data)) +#endif /* SLJIT_MALLOC_EXEC */ + +#ifndef SLJIT_FREE_EXEC +#define SLJIT_FREE_EXEC(ptr, exec_allocator_data) SLJIT_BUILTIN_FREE_EXEC((ptr), (exec_allocator_data)) +#endif /* SLJIT_FREE_EXEC */ #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR) SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr); @@ -623,7 +629,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr); #define SLJIT_EXEC_OFFSET(ptr) 0 #endif -#endif +#endif /* SLJIT_EXECUTABLE_ALLOCATOR */ /**********************************************/ /* Registers and locals offset determination. */ @@ -791,4 +797,4 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr); } /* extern "C" */ #endif -#endif +#endif /* SLJIT_CONFIG_INTERNAL_H_ */ diff --git a/src/sljit/sljitExecAllocator.c b/src/sljit/sljitExecAllocator.c index 7653907..61a32f2 100644 --- a/src/sljit/sljitExecAllocator.c +++ b/src/sljit/sljitExecAllocator.c @@ -72,9 +72,8 @@ alloc_chunk / free_chunk : * allocate executable system memory chunks * the size is always divisible by CHUNK_SIZE - allocator_grab_lock / allocator_release_lock : - * make the allocator thread safe - * can be empty if the OS (or the application) does not support threading + SLJIT_ALLOCATOR_LOCK / SLJIT_ALLOCATOR_UNLOCK : + * provided as part of sljitUtils * only the allocator requires this lock, sljit is fully thread safe as it only uses local variables */ @@ -95,6 +94,7 @@ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) #else #ifdef __APPLE__ +#ifdef MAP_ANON /* Configures TARGET_OS_OSX when appropriate */ #include @@ -104,17 +104,23 @@ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) #ifdef MAP_JIT +/* + On macOS systems, returns MAP_JIT if it is defined _and_ we're running on a + version where it's OK to have more than one JIT block. + On non-macOS systems, returns MAP_JIT if it is defined. +*/ static SLJIT_INLINE int get_map_jit_flag() { -/* On macOS systems, returns MAP_JIT if it is defined _and_ we're running on a version - of macOS where it's OK to have more than one JIT block. - On non-macOS systems, returns MAP_JIT if it is defined. */ #if TARGET_OS_OSX + sljit_sw page_size = get_page_alignment() + 1; + void *ptr; static int map_jit_flag = -1; - /* The following code is thread safe because multiple initialization - sets map_jit_flag to the same value and the code has no side-effects. - Changing the kernel version witout system restart is (very) unlikely. */ + /* + The following code is thread safe because multiple initialization + sets map_jit_flag to the same value and the code has no side-effects. + Changing the kernel version witout system restart is (very) unlikely. + */ if (map_jit_flag == -1) { struct utsname name; @@ -123,15 +129,9 @@ static SLJIT_INLINE int get_map_jit_flag() /* Kernel version for 10.14.0 (Mojave) */ if (atoi(name.release) >= 18) { - /* Only use MAP_JIT if a hardened runtime is used, because MAP_JIT is incompatible with fork(). */ + /* Only use MAP_JIT if a hardened runtime is used */ - /* mirroring page size detection from sljit_allocate_stack */ - long page_size = sysconf(_SC_PAGESIZE); - /* Should never happen */ - if (page_size < 0) - page_size = 4096; - - void *ptr = mmap(NULL, page_size, PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON, -1, 0); + ptr = mmap(NULL, page_size, PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON, -1, 0); if (ptr == MAP_FAILED) { map_jit_flag = MAP_JIT; @@ -148,7 +148,7 @@ static SLJIT_INLINE int get_map_jit_flag() } #endif /* MAP_JIT */ - +#endif /* MAP_ANON */ #endif /* __APPLE__ */ static SLJIT_INLINE void* alloc_chunk(sljit_uw size) @@ -166,10 +166,9 @@ static SLJIT_INLINE void* alloc_chunk(sljit_uw size) retval = mmap(NULL, size, prot, flags, -1, 0); #else /* !MAP_ANON */ - if (dev_zero < 0) { - if (open_dev_zero()) - return NULL; - } + if (SLJIT_UNLIKELY((dev_zero < 0) && open_dev_zero())) + return NULL; + retval = mmap(NULL, size, prot, MAP_PRIVATE, dev_zero, 0); #endif /* MAP_ANON */ @@ -253,7 +252,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) struct free_block *free_block; sljit_uw chunk_size; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); if (size < (64 - sizeof(struct block_header))) size = (64 - sizeof(struct block_header)); size = ALIGN_SIZE(size); @@ -277,7 +276,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) } allocated_size += size; header->size = size; - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return MEM_START(header); } free_block = free_block->next; @@ -286,7 +285,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) chunk_size = (size + sizeof(struct block_header) + CHUNK_SIZE - 1) & CHUNK_MASK; header = (struct block_header*)alloc_chunk(chunk_size); if (!header) { - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return NULL; } @@ -313,7 +312,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) } next_header->size = 1; next_header->prev_size = chunk_size; - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return MEM_START(header); } @@ -322,7 +321,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) struct block_header *header; struct free_block* free_block; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); header = AS_BLOCK_HEADER(ptr, -(sljit_sw)sizeof(struct block_header)); allocated_size -= header->size; @@ -359,7 +358,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) } } - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); } SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) @@ -367,7 +366,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) struct free_block* free_block; struct free_block* next_free_block; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); free_block = free_blocks; while (free_block) { @@ -382,5 +381,5 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) } SLJIT_ASSERT((total_size && free_blocks) || (!total_size && !free_blocks)); - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); } diff --git a/src/sljit/sljitLir.c b/src/sljit/sljitLir.c index 86772cc..9d128dc 100644 --- a/src/sljit/sljitLir.c +++ b/src/sljit/sljitLir.c @@ -28,7 +28,6 @@ #ifdef _WIN32 -/* For SLJIT_CACHE_FLUSH, which can expand to FlushInstructionCache. */ #include #endif /* _WIN32 */ @@ -274,6 +273,8 @@ #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR) #include "sljitProtExecAllocator.c" +#elif (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR) +#include "sljitWXExecAllocator.c" #else #include "sljitExecAllocator.c" #endif @@ -286,6 +287,10 @@ #define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr)) #endif +#ifndef SLJIT_UPDATE_WX_FLAGS +#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec) +#endif + /* Argument checking features. */ #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) @@ -366,7 +371,7 @@ static sljit_s32 compiler_initialized = 0; static void init_compiler(void); #endif -SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data) +SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data, void *exec_allocator_data) { struct sljit_compiler *compiler = (struct sljit_compiler*)SLJIT_MALLOC(sizeof(struct sljit_compiler), allocator_data); if (!compiler) @@ -393,6 +398,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allo compiler->error = SLJIT_SUCCESS; compiler->allocator_data = allocator_data; + compiler->exec_allocator_data = exec_allocator_data; compiler->buf = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, allocator_data); compiler->abuf = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, allocator_data); @@ -485,22 +491,22 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_compiler_memory_error(struct sljit_compi } #if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2) -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code) +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data) { /* Remove thumb mode flag. */ - SLJIT_FREE_EXEC((void*)((sljit_uw)code & ~0x1)); + SLJIT_FREE_EXEC((void*)((sljit_uw)code & ~0x1), exec_allocator_data); } #elif (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL) -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code) +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data) { /* Resolve indirection. */ code = (void*)(*(sljit_uw*)code); - SLJIT_FREE_EXEC(code); + SLJIT_FREE_EXEC(code, exec_allocator_data); } #else -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code) +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data) { - SLJIT_FREE_EXEC(code); + SLJIT_FREE_EXEC(code, exec_allocator_data); } #endif @@ -2292,9 +2298,10 @@ SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void) return "unsupported"; } -SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data) +SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data, void *exec_allocator_data) { SLJIT_UNUSED_ARG(allocator_data); + SLJIT_UNUSED_ARG(exec_allocator_data); SLJIT_UNREACHABLE(); return NULL; } @@ -2342,9 +2349,10 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) return 0; } -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code) +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data) { SLJIT_UNUSED_ARG(code); + SLJIT_UNUSED_ARG(exec_allocator_data); SLJIT_UNREACHABLE(); } diff --git a/src/sljit/sljitLir.h b/src/sljit/sljitLir.h index 72595bb..19b13fc 100644 --- a/src/sljit/sljitLir.h +++ b/src/sljit/sljitLir.h @@ -24,8 +24,8 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _SLJIT_LIR_H_ -#define _SLJIT_LIR_H_ +#ifndef SLJIT_LIR_H_ +#define SLJIT_LIR_H_ /* ------------------------------------------------------------------------ @@ -70,9 +70,11 @@ - pass --smc-check=all argument to valgrind, since JIT is a "self-modifying code" */ -#if !(defined SLJIT_NO_DEFAULT_CONFIG && SLJIT_NO_DEFAULT_CONFIG) +#if (defined SLJIT_HAVE_CONFIG_PRE && SLJIT_HAVE_CONFIG_PRE) +#include "sljitConfigPre.h" +#endif /* SLJIT_HAVE_CONFIG_PRE */ + #include "sljitConfig.h" -#endif /* The following header file defines useful macros for fine tuning sljit based code generators. They are listed in the beginning @@ -80,6 +82,10 @@ of sljitConfigInternal.h */ #include "sljitConfigInternal.h" +#if (defined SLJIT_HAVE_CONFIG_POST && SLJIT_HAVE_CONFIG_POST) +#include "sljitConfigPost.h" +#endif /* SLJIT_HAVE_CONFIG_POST */ + #ifdef __cplusplus extern "C" { #endif @@ -385,6 +391,7 @@ struct sljit_compiler { struct sljit_put_label *last_put_label; void *allocator_data; + void *exec_allocator_data; struct sljit_memory_fragment *buf; struct sljit_memory_fragment *abuf; @@ -485,10 +492,12 @@ struct sljit_compiler { custom memory managers. This pointer is passed to SLJIT_MALLOC and SLJIT_FREE macros. Most allocators (including the default one) ignores this value, and it is recommended to pass NULL - as a dummy value for allocator_data. + as a dummy value for allocator_data. The exec_allocator_data + has the same purpose but this one is passed to SLJIT_MALLOC_EXEC / + SLJIT_MALLOC_FREE functions. Returns NULL if failed. */ -SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data); +SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data, void *exec_allocator_data); /* Frees everything except the compiled machine code. */ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_compiler(struct sljit_compiler *compiler); @@ -535,7 +544,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil /* Free executable code. */ -SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code); +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data); /* When the protected executable allocator is used the JIT code is mapped @@ -699,7 +708,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler * */ /* - IMPORATNT NOTE: memory access MUST be naturally aligned except + IMPORTANT NOTE: memory access MUST be naturally aligned unless SLJIT_UNALIGNED macro is defined and its value is 1. length | alignment @@ -1405,12 +1414,6 @@ SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void); /* Portable helper function to get an offset of a member. */ #define SLJIT_OFFSETOF(base, member) ((sljit_sw)(&((base*)0x10)->member) - 0x10) -#if (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) -/* This global lock is useful to compile common functions. */ -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_grab_lock(void); -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void); -#endif - #if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) /* The sljit_stack structure and its manipulation functions provides @@ -1538,4 +1541,4 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_current_flags(struct sljit_compiler *com } /* extern "C" */ #endif -#endif /* _SLJIT_LIR_H_ */ +#endif /* SLJIT_LIR_H_ */ diff --git a/src/sljit/sljitNativeARM_32.c b/src/sljit/sljitNativeARM_32.c index 5d180c2..ae8479f 100644 --- a/src/sljit/sljitNativeARM_32.c +++ b/src/sljit/sljitNativeARM_32.c @@ -467,18 +467,28 @@ static SLJIT_INLINE void inline_set_jump_addr(sljit_uw jump_ptr, sljit_sw execut sljit_s32 bl = (mov_pc & 0x0000f000) != RD(TMP_PC); sljit_sw diff = (sljit_sw)(((sljit_sw)new_addr - (sljit_sw)(inst + 2) - executable_offset) >> 2); + SLJIT_UNUSED_ARG(executable_offset); + if (diff <= 0x7fffff && diff >= -0x800000) { /* Turn to branch. */ if (!bl) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0); + } inst[0] = (mov_pc & COND_MASK) | (B - CONDITIONAL) | (diff & 0xffffff); if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } } else { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + } inst[0] = (mov_pc & COND_MASK) | (BL - CONDITIONAL) | (diff & 0xffffff); inst[1] = NOP; if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } @@ -491,28 +501,52 @@ static SLJIT_INLINE void inline_set_jump_addr(sljit_uw jump_ptr, sljit_sw execut ptr = inst + 1; if (*inst != mov_pc) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + (!bl ? 1 : 2), 0); + } inst[0] = mov_pc; if (!bl) { if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } } else { inst[1] = BLX | RM(TMP_REG1); if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } } } + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 0); + } + *ptr = new_addr; + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 1); + } } #else sljit_uw *inst = (sljit_uw*)jump_ptr; + + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_ASSERT((inst[0] & 0xfff00000) == MOVW && (inst[1] & 0xfff00000) == MOVT); + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + } + inst[0] = MOVW | (inst[0] & 0xf000) | ((new_addr << 4) & 0xf0000) | (new_addr & 0xfff); inst[1] = MOVT | (inst[1] & 0xf000) | ((new_addr >> 12) & 0xf0000) | ((new_addr >> 16) & 0xfff); + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } @@ -529,10 +563,18 @@ static SLJIT_INLINE void inline_set_const(sljit_uw addr, sljit_sw executable_off sljit_uw ldr_literal = ptr[1]; sljit_uw src2; + SLJIT_UNUSED_ARG(executable_offset); + src2 = get_imm(new_constant); if (src2) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0); + } + *inst = 0xe3a00000 | (ldr_literal & 0xf000) | src2; + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } @@ -541,8 +583,14 @@ static SLJIT_INLINE void inline_set_const(sljit_uw addr, sljit_sw executable_off src2 = get_imm(~new_constant); if (src2) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0); + } + *inst = 0xe3e00000 | (ldr_literal & 0xf000) | src2; + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } @@ -555,19 +603,44 @@ static SLJIT_INLINE void inline_set_const(sljit_uw addr, sljit_sw executable_off ptr = inst + 1; if (*inst != ldr_literal) { + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 0); + } + *inst = ldr_literal; + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 1, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 1); } } + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 0); + } + *ptr = new_constant; + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(ptr, ptr + 1, 1); + } #else sljit_uw *inst = (sljit_uw*)addr; + + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_ASSERT((inst[0] & 0xfff00000) == MOVW && (inst[1] & 0xfff00000) == MOVT); + + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); + } + inst[0] = MOVW | (inst[0] & 0xf000) | ((new_constant << 4) & 0xf0000) | (new_constant & 0xfff); inst[1] = MOVT | (inst[1] & 0xf000) | ((new_constant >> 12) & 0xf0000) | ((new_constant >> 16) & 0xfff); + if (flush_cache) { + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } @@ -612,7 +685,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil #else size = compiler->size; #endif - code = (sljit_uw*)SLJIT_MALLOC_EXEC(size * sizeof(sljit_uw)); + code = (sljit_uw*)SLJIT_MALLOC_EXEC(size * sizeof(sljit_uw), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -653,7 +726,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil } else { if (SLJIT_UNLIKELY(resolve_const_pool_index(compiler, &first_patch, cpool_current_index, cpool_start_address, buf_ptr))) { - SLJIT_FREE_EXEC(code); + SLJIT_FREE_EXEC(code, compiler->exec_allocator_data); compiler->error = SLJIT_ERR_ALLOC_FAILED; return NULL; } @@ -756,7 +829,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil cpool_current_index = 0; while (buf_ptr < buf_end) { if (SLJIT_UNLIKELY(resolve_const_pool_index(compiler, &first_patch, cpool_current_index, cpool_start_address, buf_ptr))) { - SLJIT_FREE_EXEC(code); + SLJIT_FREE_EXEC(code, compiler->exec_allocator_data); compiler->error = SLJIT_ERR_ALLOC_FAILED; return NULL; } @@ -856,6 +929,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_uw *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } diff --git a/src/sljit/sljitNativeARM_64.c b/src/sljit/sljitNativeARM_64.c index eaca095..52267e7 100644 --- a/src/sljit/sljitNativeARM_64.c +++ b/src/sljit/sljitNativeARM_64.c @@ -151,16 +151,6 @@ static SLJIT_INLINE sljit_s32 emit_imm64_const(struct sljit_compiler *compiler, return push_inst(compiler, MOVK | RD(dst) | ((imm >> 48) << 5) | (3 << 21)); } -static SLJIT_INLINE void modify_imm64_const(sljit_ins* inst, sljit_uw new_imm) -{ - sljit_s32 dst = inst[0] & 0x1f; - SLJIT_ASSERT((inst[0] & 0xffe00000) == MOVZ && (inst[1] & 0xffe00000) == (MOVK | (1 << 21))); - inst[0] = MOVZ | dst | ((new_imm & 0xffff) << 5); - inst[1] = MOVK | dst | (((new_imm >> 16) & 0xffff) << 5) | (1 << 21); - inst[2] = MOVK | dst | (((new_imm >> 32) & 0xffff) << 5) | (2 << 21); - inst[3] = MOVK | dst | ((new_imm >> 48) << 5) | (3 << 21); -} - static SLJIT_INLINE sljit_sw detect_jump_type(struct sljit_jump *jump, sljit_ins *code_ptr, sljit_ins *code, sljit_sw executable_offset) { sljit_sw diff; @@ -253,7 +243,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -380,6 +370,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } @@ -2034,15 +2025,24 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct slj SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins* inst = (sljit_ins*)addr; - modify_imm64_const(inst, new_target); + sljit_s32 dst; + SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0); + + dst = inst[0] & 0x1f; + SLJIT_ASSERT((inst[0] & 0xffe00000) == MOVZ && (inst[1] & 0xffe00000) == (MOVK | (1 << 21))); + inst[0] = MOVZ | dst | ((new_target & 0xffff) << 5); + inst[1] = MOVK | dst | (((new_target >> 16) & 0xffff) << 5) | (1 << 21); + inst[2] = MOVK | dst | (((new_target >> 32) & 0xffff) << 5) | (2 << 21); + inst[3] = MOVK | dst | ((new_target >> 48) << 5) | (3 << 21); + + SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 4); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins* inst = (sljit_ins*)addr; - modify_imm64_const(inst, new_constant); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 4); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/src/sljit/sljitNativeARM_T2_32.c b/src/sljit/sljitNativeARM_T2_32.c index a81e008..4624882 100644 --- a/src/sljit/sljitNativeARM_T2_32.c +++ b/src/sljit/sljitNativeARM_T2_32.c @@ -377,7 +377,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_u16*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_u16)); + code = (sljit_u16*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_u16), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -463,6 +463,8 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); + /* Set thumb mode flag. */ return (void*)((sljit_uw)code | 0x1); } @@ -608,7 +610,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s Although some clever things could be done here, "NOT IMM" does not worth the efforts. */ break; case SLJIT_ADD: - nimm = -imm; + nimm = -(sljit_sw)imm; if (IS_2_LO_REGS(reg, dst)) { if (imm <= 0x7) return push_inst16(compiler, ADDSI3 | IMM3(imm) | RD3(dst) | RN3(reg)); @@ -630,7 +632,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s nimm = get_imm(imm); if (nimm != INVALID_IMM) return push_inst32(compiler, ADD_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | nimm); - nimm = get_imm(-imm); + nimm = get_imm(-(sljit_sw)imm); if (nimm != INVALID_IMM) return push_inst32(compiler, SUB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | nimm); break; @@ -655,11 +657,11 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s nimm = get_imm(imm); if (nimm != INVALID_IMM) return push_inst32(compiler, CMPI_W | RN4(reg) | nimm); - nimm = get_imm(-imm); + nimm = get_imm(-(sljit_sw)imm); if (nimm != INVALID_IMM) return push_inst32(compiler, CMNI_W | RN4(reg) | nimm); } - nimm = -imm; + nimm = -(sljit_sw)imm; if (IS_2_LO_REGS(reg, dst)) { if (imm <= 0x7) return push_inst16(compiler, SUBSI3 | IMM3(imm) | RD3(dst) | RN3(reg)); @@ -681,7 +683,7 @@ static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, s nimm = get_imm(imm); if (nimm != INVALID_IMM) return push_inst32(compiler, SUB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | nimm); - nimm = get_imm(-imm); + nimm = get_imm(-(sljit_sw)imm); if (nimm != INVALID_IMM) return push_inst32(compiler, ADD_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | nimm); break; @@ -2366,15 +2368,16 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct slj SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_u16 *inst = (sljit_u16*)addr; + SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0); modify_imm32_const(inst, new_target); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1); inst = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 4); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_u16 *inst = (sljit_u16*)addr; - modify_imm32_const(inst, new_constant); - inst = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 4); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/src/sljit/sljitNativeMIPS_32.c b/src/sljit/sljitNativeMIPS_32.c index 777627b..f887ee1 100644 --- a/src/sljit/sljitNativeMIPS_32.c +++ b/src/sljit/sljitNativeMIPS_32.c @@ -425,23 +425,20 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins *)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); SLJIT_ASSERT((inst[0] & 0xffe00000) == LUI && (inst[1] & 0xfc000000) == ORI); inst[0] = (inst[0] & 0xffff0000) | ((new_target >> 16) & 0xffff); inst[1] = (inst[1] & 0xffff0000) | (new_target & 0xffff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins *)addr; - - SLJIT_ASSERT((inst[0] & 0xffe00000) == LUI && (inst[1] & 0xfc000000) == ORI); - inst[0] = (inst[0] & 0xffff0000) | ((new_constant >> 16) & 0xffff); - inst[1] = (inst[1] & 0xffff0000) | (new_constant & 0xffff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 2); + sljit_set_jump_addr(addr, new_constant, executable_offset); } static sljit_s32 call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_ins *ins_ptr) diff --git a/src/sljit/sljitNativeMIPS_64.c b/src/sljit/sljitNativeMIPS_64.c index 479244d..5ab9b7d 100644 --- a/src/sljit/sljitNativeMIPS_64.c +++ b/src/sljit/sljitNativeMIPS_64.c @@ -525,25 +525,21 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins *)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 6, 0); inst[0] = (inst[0] & 0xffff0000) | ((new_target >> 48) & 0xffff); inst[1] = (inst[1] & 0xffff0000) | ((new_target >> 32) & 0xffff); inst[3] = (inst[3] & 0xffff0000) | ((new_target >> 16) & 0xffff); inst[5] = (inst[5] & 0xffff0000) | (new_target & 0xffff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 6, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 6); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins *)addr; - - inst[0] = (inst[0] & 0xffff0000) | ((new_constant >> 48) & 0xffff); - inst[1] = (inst[1] & 0xffff0000) | ((new_constant >> 32) & 0xffff); - inst[3] = (inst[3] & 0xffff0000) | ((new_constant >> 16) & 0xffff); - inst[5] = (inst[5] & 0xffff0000) | (new_constant & 0xffff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 6); + sljit_set_jump_addr(addr, new_constant, executable_offset); } static sljit_s32 call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_ins *ins_ptr) diff --git a/src/sljit/sljitNativeMIPS_common.c b/src/sljit/sljitNativeMIPS_common.c index 88df904..8b8c12a 100644 --- a/src/sljit/sljitNativeMIPS_common.c +++ b/src/sljit/sljitNativeMIPS_common.c @@ -290,9 +290,9 @@ static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 4] = { Useful for reordering instructions in the delay slot. */ static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins, sljit_s32 delay_slot) { + sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins)); SLJIT_ASSERT(delay_slot == MOVABLE_INS || delay_slot >= UNMOVABLE_INS || delay_slot == ((ins >> 11) & 0x1f) || delay_slot == ((ins >> 16) & 0x1f)); - sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins)); FAIL_IF(!ptr); *ptr = ins; compiler->size++; @@ -520,7 +520,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -667,6 +667,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil /* GCC workaround for invalid code generation with -O2. */ sljit_cache_flush(code, code_ptr); #endif + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } @@ -679,7 +680,7 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) #ifdef SLJIT_IS_FPU_AVAILABLE return SLJIT_IS_FPU_AVAILABLE; #elif defined(__GNUC__) - asm ("cfc1 %0, $0" : "=r"(fir)); + __asm__ ("cfc1 %0, $0" : "=r"(fir)); return (fir >> 22) & 0x1; #else #error "FIR check is not implemented for this architecture" diff --git a/src/sljit/sljitNativePPC_32.c b/src/sljit/sljitNativePPC_32.c index 3ce7411..7d9ec53 100644 --- a/src/sljit/sljitNativePPC_32.c +++ b/src/sljit/sljitNativePPC_32.c @@ -258,21 +258,18 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins *)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); SLJIT_ASSERT((inst[0] & 0xfc1f0000) == ADDIS && (inst[1] & 0xfc000000) == ORI); inst[0] = (inst[0] & 0xffff0000) | ((new_target >> 16) & 0xffff); inst[1] = (inst[1] & 0xffff0000) | (new_target & 0xffff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins *)addr; - - SLJIT_ASSERT((inst[0] & 0xfc1f0000) == ADDIS && (inst[1] & 0xfc000000) == ORI); - inst[0] = (inst[0] & 0xffff0000) | ((new_constant >> 16) & 0xffff); - inst[1] = (inst[1] & 0xffff0000) | (new_constant & 0xffff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 2); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/src/sljit/sljitNativePPC_64.c b/src/sljit/sljitNativePPC_64.c index 3b73021..92147d2 100644 --- a/src/sljit/sljitNativePPC_64.c +++ b/src/sljit/sljitNativePPC_64.c @@ -477,23 +477,19 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins*)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 5, 0); inst[0] = (inst[0] & 0xffff0000) | ((new_target >> 48) & 0xffff); inst[1] = (inst[1] & 0xffff0000) | ((new_target >> 32) & 0xffff); inst[3] = (inst[3] & 0xffff0000) | ((new_target >> 16) & 0xffff); inst[4] = (inst[4] & 0xffff0000) | (new_target & 0xffff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 5, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 5); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins*)addr; - - inst[0] = (inst[0] & 0xffff0000) | ((new_constant >> 48) & 0xffff); - inst[1] = (inst[1] & 0xffff0000) | ((new_constant >> 32) & 0xffff); - inst[3] = (inst[3] & 0xffff0000) | ((new_constant >> 16) & 0xffff); - inst[4] = (inst[4] & 0xffff0000) | (new_constant & 0xffff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 5); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/src/sljit/sljitNativePPC_common.c b/src/sljit/sljitNativePPC_common.c index 590f91c..d84562c 100644 --- a/src/sljit/sljitNativePPC_common.c +++ b/src/sljit/sljitNativePPC_common.c @@ -404,7 +404,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil compiler->size += (sizeof(struct sljit_function_context) / sizeof(sljit_ins)); #endif #endif - code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -607,6 +607,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); #if (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL) return code_ptr; diff --git a/src/sljit/sljitNativeSPARC_32.c b/src/sljit/sljitNativeSPARC_32.c index 8079fad..e5167f0 100644 --- a/src/sljit/sljitNativeSPARC_32.c +++ b/src/sljit/sljitNativeSPARC_32.c @@ -266,21 +266,18 @@ static SLJIT_INLINE sljit_s32 emit_const(struct sljit_compiler *compiler, sljit_ SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { sljit_ins *inst = (sljit_ins *)addr; + SLJIT_UNUSED_ARG(executable_offset); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 0); SLJIT_ASSERT(((inst[0] & 0xc1c00000) == 0x01000000) && ((inst[1] & 0xc1f82000) == 0x80102000)); inst[0] = (inst[0] & 0xffc00000) | ((new_target >> 10) & 0x3fffff); inst[1] = (inst[1] & 0xfffffc00) | (new_target & 0x3ff); + SLJIT_UPDATE_WX_FLAGS(inst, inst + 2, 1); inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); SLJIT_CACHE_FLUSH(inst, inst + 2); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { - sljit_ins *inst = (sljit_ins *)addr; - - SLJIT_ASSERT(((inst[0] & 0xc1c00000) == 0x01000000) && ((inst[1] & 0xc1f82000) == 0x80102000)); - inst[0] = (inst[0] & 0xffc00000) | ((new_constant >> 10) & 0x3fffff); - inst[1] = (inst[1] & 0xfffffc00) | (new_constant & 0x3ff); - inst = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset); - SLJIT_CACHE_FLUSH(inst, inst + 2); + sljit_set_jump_addr(addr, new_constant, executable_offset); } diff --git a/src/sljit/sljitNativeSPARC_common.c b/src/sljit/sljitNativeSPARC_common.c index 7d6be6c..544d80d 100644 --- a/src/sljit/sljitNativeSPARC_common.c +++ b/src/sljit/sljitNativeSPARC_common.c @@ -311,7 +311,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -437,6 +437,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset); SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } diff --git a/src/sljit/sljitNativeTILEGX_64.c b/src/sljit/sljitNativeTILEGX_64.c index d69ecd6..4e3e9c6 100644 --- a/src/sljit/sljitNativeTILEGX_64.c +++ b/src/sljit/sljitNativeTILEGX_64.c @@ -1011,7 +1011,7 @@ SLJIT_API_FUNC_ATTRIBUTE void * sljit_generate_code(struct sljit_compiler *compi CHECK_PTR(check_sljit_generate_code(compiler)); reverse_buf(compiler); - code = (sljit_ins *)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins)); + code = (sljit_ins *)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -1113,7 +1113,9 @@ SLJIT_API_FUNC_ATTRIBUTE void * sljit_generate_code(struct sljit_compiler *compi compiler->error = SLJIT_ERR_COMPILED; compiler->executable_size = (code_ptr - code) * sizeof(sljit_ins); + SLJIT_CACHE_FLUSH(code, code_ptr); + SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1); return code; } diff --git a/src/sljit/sljitNativeX86_common.c b/src/sljit/sljitNativeX86_common.c index 74965e3..ddcc5eb 100644 --- a/src/sljit/sljitNativeX86_common.c +++ b/src/sljit/sljitNativeX86_common.c @@ -506,7 +506,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil reverse_buf(compiler); /* Second code generation pass. */ - code = (sljit_u8*)SLJIT_MALLOC_EXEC(compiler->size); + code = (sljit_u8*)SLJIT_MALLOC_EXEC(compiler->size, compiler->exec_allocator_data); PTR_FAIL_WITH_EXEC_IF(code); buf = compiler->buf; @@ -557,7 +557,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil SLJIT_ASSERT(put_label->label); put_label->addr = (sljit_uw)code_ptr; #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) - code_ptr = generate_put_label_code(put_label, code_ptr, (sljit_uw)(SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size)); + code_ptr = generate_put_label_code(put_label, code_ptr, (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size); #endif put_label = put_label->next; break; @@ -629,7 +629,11 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compil compiler->error = SLJIT_ERR_COMPILED; compiler->executable_offset = executable_offset; compiler->executable_size = code_ptr - code; - return (void*)(code + executable_offset); + + code = (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code, executable_offset); + + SLJIT_UPDATE_WX_FLAGS(code, (sljit_u8*)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset), 1); + return (void*)code; } SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type) @@ -725,15 +729,16 @@ static SLJIT_INLINE sljit_s32 emit_endbranch(struct sljit_compiler *compiler) #else *inst = 0xfa; #endif -#else +#else /* !SLJIT_CONFIG_X86_CET */ SLJIT_UNUSED_ARG(compiler); -#endif +#endif /* SLJIT_CONFIG_X86_CET */ return SLJIT_SUCCESS; } +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__) + static SLJIT_INLINE sljit_s32 emit_rdssp(struct sljit_compiler *compiler, sljit_s32 reg) { -#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) sljit_u8 *inst; sljit_s32 size; @@ -753,16 +758,11 @@ static SLJIT_INLINE sljit_s32 emit_rdssp(struct sljit_compiler *compiler, sljit_ *inst++ = 0x0f; *inst++ = 0x1e; *inst = (0x3 << 6) | (0x1 << 3) | (reg_map[reg] & 0x7); -#else - SLJIT_UNUSED_ARG(compiler); - SLJIT_UNUSED_ARG(reg); -#endif return SLJIT_SUCCESS; } static SLJIT_INLINE sljit_s32 emit_incssp(struct sljit_compiler *compiler, sljit_s32 reg) { -#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) sljit_u8 *inst; sljit_s32 size; @@ -782,29 +782,28 @@ static SLJIT_INLINE sljit_s32 emit_incssp(struct sljit_compiler *compiler, sljit *inst++ = 0x0f; *inst++ = 0xae; *inst = (0x3 << 6) | (0x5 << 3) | (reg_map[reg] & 0x7); -#else - SLJIT_UNUSED_ARG(compiler); - SLJIT_UNUSED_ARG(reg); -#endif return SLJIT_SUCCESS; } +#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */ + static SLJIT_INLINE sljit_s32 cpu_has_shadow_stack(void) { -#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__) return _get_ssp() != 0; -#else +#else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */ return 0; -#endif +#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */ } static SLJIT_INLINE sljit_s32 adjust_shadow_stack(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw, sljit_s32 base, sljit_sw disp) { -#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) - sljit_u8 *inst; +#if (defined SLJIT_CONFIG_X86_CET && SLJIT_CONFIG_X86_CET) && defined (__SHSTK__) + sljit_u8 *inst, *jz_after_cmp_inst; + sljit_uw size_jz_after_cmp_inst; - sljit_s32 size_before_rdssp_inst = compiler->size; + sljit_uw size_before_rdssp_inst = compiler->size; /* Generate "RDSSP TMP_REG1". */ FAIL_IF(emit_rdssp(compiler, TMP_REG1)); @@ -839,8 +838,8 @@ static SLJIT_INLINE sljit_s32 adjust_shadow_stack(struct sljit_compiler *compile FAIL_IF(!inst); INC_SIZE(2); *inst++ = get_jump_code(SLJIT_EQUAL) - 0x10; - sljit_uw size_jz_after_cmp_inst = compiler->size; - sljit_u8 *jz_after_cmp_inst = inst; + size_jz_after_cmp_inst = compiler->size; + jz_after_cmp_inst = inst; #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) /* REX_W is not necessary. */ @@ -860,13 +859,13 @@ static SLJIT_INLINE sljit_s32 adjust_shadow_stack(struct sljit_compiler *compile *inst = size_before_rdssp_inst - compiler->size; *jz_after_cmp_inst = compiler->size - size_jz_after_cmp_inst; -#else /* SLJIT_CONFIG_X86_CET */ +#else /* !SLJIT_CONFIG_X86_CET || !__SHSTK__ */ SLJIT_UNUSED_ARG(compiler); SLJIT_UNUSED_ARG(src); SLJIT_UNUSED_ARG(srcw); SLJIT_UNUSED_ARG(base); SLJIT_UNUSED_ARG(disp); -#endif /* SLJIT_CONFIG_X86_CET */ +#endif /* SLJIT_CONFIG_X86_CET && __SHSTK__ */ return SLJIT_SUCCESS; } @@ -3123,15 +3122,21 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct slj SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset) { SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 0); #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) sljit_unaligned_store_sw((void*)addr, new_target - (addr + 4) - (sljit_uw)executable_offset); #else sljit_unaligned_store_sw((void*)addr, (sljit_sw) new_target); #endif + SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_uw)), 1); } SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset) { SLJIT_UNUSED_ARG(executable_offset); + + SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 0); sljit_unaligned_store_sw((void*)addr, new_constant); + SLJIT_UPDATE_WX_FLAGS((void*)addr, (void*)(addr + sizeof(sljit_sw)), 1); } diff --git a/src/sljit/sljitProtExecAllocator.c b/src/sljit/sljitProtExecAllocator.c index 3f412fe..147175a 100644 --- a/src/sljit/sljitProtExecAllocator.c +++ b/src/sljit/sljitProtExecAllocator.c @@ -76,105 +76,106 @@ struct chunk_header { alloc_chunk / free_chunk : * allocate executable system memory chunks * the size is always divisible by CHUNK_SIZE - allocator_grab_lock / allocator_release_lock : - * make the allocator thread safe - * can be empty if the OS (or the application) does not support threading + SLJIT_ALLOCATOR_LOCK / SLJIT_ALLOCATOR_UNLOCK : + * provided as part of sljitUtils * only the allocator requires this lock, sljit is fully thread safe as it only uses local variables */ +#ifndef __NetBSD__ +#include #include +#include +#include #ifndef O_NOATIME #define O_NOATIME 0 #endif -#ifdef __O_TMPFILE +/* this is a linux extension available since kernel 3.11 */ #ifndef O_TMPFILE -#define O_TMPFILE (__O_TMPFILE | O_DIRECTORY) +#define O_TMPFILE 020200000 #endif -#endif - -#if !(defined(__NetBSD__) && defined(MAP_REMAPDUP)) -int mkostemp(char *template, int flags); -#ifdef __NetBSD__ -/* - * this is a workaround for NetBSD < 8 that lacks a system provided - * secure_getenv function. - * ideally this should never be used, as the standard allocator is - * a preferred option for those systems and should be used instead. - */ -#define secure_getenv(name) issetugid() ? NULL : getenv(name) -#else +#ifndef _GNU_SOURCE char *secure_getenv(const char *name); +int mkostemp(char *template, int flags); #endif static SLJIT_INLINE int create_tempfile(void) { int fd; - char tmp_name[256]; - size_t tmp_name_len; + size_t tmp_name_len = 0; char *dir; - size_t len; + struct stat st; +#if defined(SLJIT_SINGLE_THREADED) && SLJIT_SINGLE_THREADED + mode_t mode; +#endif #ifdef HAVE_MEMFD_CREATE /* this is a GNU extension, make sure to use -D_GNU_SOURCE */ fd = memfd_create("sljit", MFD_CLOEXEC); - if (fd != -1) + if (fd != -1) { + fchmod(fd, 0); return fd; -#endif - -#ifdef P_tmpdir - len = (P_tmpdir != NULL) ? strlen(P_tmpdir) : 0; - - if (len > 0 && len < sizeof(tmp_name)) { - strcpy(tmp_name, P_tmpdir); - tmp_name_len = len; } - else { - strcpy(tmp_name, "/tmp"); - tmp_name_len = 4; - } -#else - strcpy(tmp_name, "/tmp"); - tmp_name_len = 4; #endif dir = secure_getenv("TMPDIR"); if (dir) { - len = strlen(dir); - if (len > 0 && len < sizeof(tmp_name)) { - strcpy(tmp_name, dir); - tmp_name_len = len; + tmp_name_len = strlen(dir); + if (tmp_name_len > 0 && tmp_name_len < sizeof(tmp_name)) { + if ((stat(dir, &st) == 0) && S_ISDIR(st.st_mode)) + strcpy(tmp_name, dir); } } +#ifdef P_tmpdir + if (!tmp_name_len) { + tmp_name_len = strlen(P_tmpdir); + if (tmp_name_len > 0 && tmp_name_len < sizeof(tmp_name)) + strcpy(tmp_name, P_tmpdir); + } +#endif + if (!tmp_name_len) { + strcpy(tmp_name, "/tmp"); + tmp_name_len = 4; + } + SLJIT_ASSERT(tmp_name_len > 0 && tmp_name_len < sizeof(tmp_name)); - while (tmp_name_len > 0 && tmp_name[tmp_name_len - 1] == '/') { - tmp_name_len--; - tmp_name[tmp_name_len] = '\0'; - } + if (tmp_name[tmp_name_len - 1] == '/') + tmp_name[--tmp_name_len] = '\0'; -#ifdef O_TMPFILE - fd = open(tmp_name, O_TMPFILE | O_EXCL | O_RDWR | O_NOATIME | O_CLOEXEC, S_IRUSR | S_IWUSR); +#ifdef __linux__ + /* + * the previous trimming might had left an empty string if TMPDIR="/" + * so work around the problem below + */ + fd = open(tmp_name_len ? tmp_name : "/", + O_TMPFILE | O_EXCL | O_RDWR | O_NOATIME | O_CLOEXEC, 0); if (fd != -1) return fd; #endif if (tmp_name_len + 7 >= sizeof(tmp_name)) - { return -1; - } strcpy(tmp_name + tmp_name_len, "/XXXXXX"); +#if defined(SLJIT_SINGLE_THREADED) && SLJIT_SINGLE_THREADED + mode = umask(0777); +#endif fd = mkostemp(tmp_name, O_CLOEXEC | O_NOATIME); +#if defined(SLJIT_SINGLE_THREADED) && SLJIT_SINGLE_THREADED + umask(mode); +#else + fchmod(fd, 0); +#endif if (fd == -1) - return fd; + return -1; if (unlink(tmp_name)) { close(fd); @@ -217,34 +218,36 @@ static SLJIT_INLINE struct chunk_header* alloc_chunk(sljit_uw size) return retval; } #else +/* + * MAP_REMAPDUP is a NetBSD extension available sinde 8.0, make sure to + * adjust your feature macros (ex: -D_NETBSD_SOURCE) as needed + */ static SLJIT_INLINE struct chunk_header* alloc_chunk(sljit_uw size) { struct chunk_header *retval; - void *maprx; retval = (struct chunk_header *)mmap(NULL, size, - PROT_MPROTECT(PROT_EXEC|PROT_WRITE|PROT_READ), - MAP_ANON, -1, 0); + PROT_READ | PROT_WRITE | PROT_MPROTECT(PROT_EXEC), + MAP_ANON | MAP_SHARED, -1, 0); if (retval == MAP_FAILED) return NULL; - maprx = mremap(retval, size, NULL, size, MAP_REMAPDUP); - if (maprx == MAP_FAILED) { + retval->executable = mremap(retval, size, NULL, size, MAP_REMAPDUP); + if (retval->executable == MAP_FAILED) { munmap((void *)retval, size); return NULL; } - if (mprotect(retval, size, PROT_READ | PROT_WRITE) == -1 || - mprotect(maprx, size, PROT_READ | PROT_EXEC) == -1) { - munmap(maprx, size); + if (mprotect(retval->executable, size, PROT_READ | PROT_EXEC) == -1) { + munmap(retval->executable, size); munmap((void *)retval, size); return NULL; } - retval->executable = maprx; + return retval; } -#endif /* NetBSD >= 8 */ +#endif /* NetBSD */ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) { @@ -318,7 +321,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) sljit_uw chunk_size; sljit_sw executable_offset; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); if (size < (64 - sizeof(struct block_header))) size = (64 - sizeof(struct block_header)); size = ALIGN_SIZE(size); @@ -343,7 +346,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) } allocated_size += size; header->size = size; - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return MEM_START(header); } free_block = free_block->next; @@ -354,7 +357,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) chunk_header = alloc_chunk(chunk_size); if (!chunk_header) { - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return NULL; } @@ -388,7 +391,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) next_header->size = 1; next_header->prev_size = chunk_size; next_header->executable_offset = executable_offset; - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); return MEM_START(header); } @@ -397,7 +400,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) struct block_header *header; struct free_block* free_block; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); header = AS_BLOCK_HEADER(ptr, -(sljit_sw)sizeof(struct block_header)); header = AS_BLOCK_HEADER(header, -header->executable_offset); allocated_size -= header->size; @@ -437,7 +440,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) } } - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); } SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) @@ -445,7 +448,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) struct free_block* free_block; struct free_block* next_free_block; - allocator_grab_lock(); + SLJIT_ALLOCATOR_LOCK(); free_block = free_blocks; while (free_block) { @@ -462,7 +465,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) } SLJIT_ASSERT((total_size && free_blocks) || (!total_size && !free_blocks)); - allocator_release_lock(); + SLJIT_ALLOCATOR_UNLOCK(); } SLJIT_API_FUNC_ATTRIBUTE sljit_sw sljit_exec_offset(void* ptr) diff --git a/src/sljit/sljitUtils.c b/src/sljit/sljitUtils.c index 0276fa1..08ca35c 100644 --- a/src/sljit/sljitUtils.c +++ b/src/sljit/sljitUtils.c @@ -28,131 +28,50 @@ /* Locks */ /* ------------------------------------------------------------------------ */ -#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) || (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) +/* Executable Allocator */ +#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) \ + && !(defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR) #if (defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED) - -#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) - -static SLJIT_INLINE void allocator_grab_lock(void) -{ - /* Always successful. */ -} - -static SLJIT_INLINE void allocator_release_lock(void) -{ - /* Always successful. */ -} - -#endif /* SLJIT_EXECUTABLE_ALLOCATOR */ - -#if (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) - -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_grab_lock(void) -{ - /* Always successful. */ -} - -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void) -{ - /* Always successful. */ -} - -#endif /* SLJIT_UTIL_GLOBAL_LOCK */ - -#elif defined(_WIN32) /* SLJIT_SINGLE_THREADED */ - -#include "windows.h" - -#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) - -static HANDLE allocator_mutex = 0; - -static SLJIT_INLINE void allocator_grab_lock(void) -{ - /* No idea what to do if an error occures. Static mutexes should never fail... */ - if (!allocator_mutex) - allocator_mutex = CreateMutex(NULL, TRUE, NULL); - else - WaitForSingleObject(allocator_mutex, INFINITE); -} - -static SLJIT_INLINE void allocator_release_lock(void) -{ - ReleaseMutex(allocator_mutex); -} - -#endif /* SLJIT_EXECUTABLE_ALLOCATOR */ - -#if (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) - -static HANDLE global_mutex = 0; - -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_grab_lock(void) -{ - /* No idea what to do if an error occures. Static mutexes should never fail... */ - if (!global_mutex) - global_mutex = CreateMutex(NULL, TRUE, NULL); - else - WaitForSingleObject(global_mutex, INFINITE); -} - -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void) -{ - ReleaseMutex(global_mutex); -} - -#endif /* SLJIT_UTIL_GLOBAL_LOCK */ - -#else /* _WIN32 */ - -#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) - +#define SLJIT_ALLOCATOR_LOCK() +#define SLJIT_ALLOCATOR_UNLOCK() +#elif !(defined _WIN32) #include -static pthread_mutex_t allocator_mutex = PTHREAD_MUTEX_INITIALIZER; - -static SLJIT_INLINE void allocator_grab_lock(void) -{ - pthread_mutex_lock(&allocator_mutex); -} +static pthread_mutex_t allocator_lock = PTHREAD_MUTEX_INITIALIZER; -static SLJIT_INLINE void allocator_release_lock(void) -{ - pthread_mutex_unlock(&allocator_mutex); -} +#define SLJIT_ALLOCATOR_LOCK() pthread_mutex_lock(&allocator_lock) +#define SLJIT_ALLOCATOR_UNLOCK() pthread_mutex_unlock(&allocator_lock) +#else /* windows */ +static HANDLE allocator_lock; -#endif /* SLJIT_EXECUTABLE_ALLOCATOR */ - -#if (defined SLJIT_UTIL_GLOBAL_LOCK && SLJIT_UTIL_GLOBAL_LOCK) - -#include - -static pthread_mutex_t global_mutex = PTHREAD_MUTEX_INITIALIZER; - -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_grab_lock(void) -{ - pthread_mutex_lock(&global_mutex); -} - -SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void) +static SLJIT_INLINE void allocator_grab_lock(void) { - pthread_mutex_unlock(&global_mutex); + HANDLE lock; + if (SLJIT_UNLIKELY(!allocator_lock)) { + lock = CreateMutex(NULL, FALSE, NULL); + if (InterlockedCompareExchangePointer(&allocator_lock, lock, NULL)) + CloseHandle(lock); + } + WaitForSingleObject(allocator_lock, INFINITE); } -#endif /* SLJIT_UTIL_GLOBAL_LOCK */ - -#endif /* _WIN32 */ +#define SLJIT_ALLOCATOR_LOCK() allocator_grab_lock() +#define SLJIT_ALLOCATOR_UNLOCK() ReleaseMutex(allocator_lock) +#endif /* thread implementation */ +#endif /* SLJIT_EXECUTABLE_ALLOCATOR && !SLJIT_WX_EXECUTABLE_ALLOCATOR */ /* ------------------------------------------------------------------------ */ /* Stack */ /* ------------------------------------------------------------------------ */ -#if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) || (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) +#if ((defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) \ + && !(defined SLJIT_UTIL_SIMPLE_STACK_ALLOCATION && SLJIT_UTIL_SIMPLE_STACK_ALLOCATION)) \ + || ((defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) \ + && !((defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR) \ + || (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR))) -#ifdef _WIN32 -#include "windows.h" -#else /* !_WIN32 */ +#ifndef _WIN32 /* Provides mmap function. */ #include #include @@ -163,56 +82,84 @@ SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_release_lock(void) #endif /* MAP_ANONYMOUS */ #endif /* !MAP_ANON */ -#ifndef MADV_DONTNEED -#ifdef POSIX_MADV_DONTNEED -#define MADV_DONTNEED POSIX_MADV_DONTNEED -#endif /* POSIX_MADV_DONTNEED */ -#endif /* !MADV_DONTNEED */ - -/* For detecting the page size. */ -#include - #ifndef MAP_ANON #include -/* Some old systems does not have MAP_ANON. */ -static sljit_s32 dev_zero = -1; +#ifdef O_CLOEXEC +#define SLJIT_CLOEXEC O_CLOEXEC +#else /* !O_CLOEXEC */ +#define SLJIT_CLOEXEC 0 +#endif /* O_CLOEXEC */ + +/* Some old systems do not have MAP_ANON. */ +static int dev_zero = -1; #if (defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED) -static SLJIT_INLINE sljit_s32 open_dev_zero(void) +static SLJIT_INLINE int open_dev_zero(void) { - dev_zero = open("/dev/zero", O_RDWR); + dev_zero = open("/dev/zero", O_RDWR | SLJIT_CLOEXEC); + return dev_zero < 0; } -#else /* SLJIT_SINGLE_THREADED */ +#else /* !SLJIT_SINGLE_THREADED */ #include static pthread_mutex_t dev_zero_mutex = PTHREAD_MUTEX_INITIALIZER; -static SLJIT_INLINE sljit_s32 open_dev_zero(void) +static SLJIT_INLINE int open_dev_zero(void) { pthread_mutex_lock(&dev_zero_mutex); - /* The dev_zero might be initialized by another thread during the waiting. */ - if (dev_zero < 0) { - dev_zero = open("/dev/zero", O_RDWR); - } + if (SLJIT_UNLIKELY(dev_zero < 0)) + dev_zero = open("/dev/zero", O_RDWR | SLJIT_CLOEXEC); + pthread_mutex_unlock(&dev_zero_mutex); return dev_zero < 0; } #endif /* SLJIT_SINGLE_THREADED */ - +#undef SLJIT_CLOEXEC #endif /* !MAP_ANON */ +#endif /* !_WIN32 */ +#endif /* open_dev_zero */ -#endif /* _WIN32 */ +#if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) \ + || (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR) -#endif /* SLJIT_UTIL_STACK || SLJIT_EXECUTABLE_ALLOCATOR */ +#ifdef _WIN32 -#endif /* SLJIT_EXECUTABLE_ALLOCATOR || SLJIT_UTIL_GLOBAL_LOCK */ +static SLJIT_INLINE sljit_sw get_page_alignment(void) { + SYSTEM_INFO si; + static sljit_sw sljit_page_align; + if (!sljit_page_align) { + GetSystemInfo(&si); + sljit_page_align = si.dwPageSize - 1; + } + return sljit_page_align; +} + +#else + +#include + +static SLJIT_INLINE sljit_sw get_page_alignment(void) { + static sljit_sw sljit_page_align; + if (!sljit_page_align) { + sljit_page_align = sysconf(_SC_PAGESIZE); + /* Should never happen. */ + if (sljit_page_align < 0) + sljit_page_align = 4096; + sljit_page_align--; + } + return sljit_page_align; +} + +#endif /* _WIN32 */ + +#endif /* get_page_alignment() */ #if (defined SLJIT_UTIL_STACK && SLJIT_UTIL_STACK) @@ -264,16 +211,6 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_u8 *SLJIT_FUNC sljit_stack_resize(struct sljit_st #ifdef _WIN32 -SLJIT_INLINE static sljit_sw get_page_alignment(void) { - SYSTEM_INFO si; - static sljit_sw sljit_page_align; - if (!sljit_page_align) { - GetSystemInfo(&si); - sljit_page_align = si.dwPageSize - 1; - } - return sljit_page_align; -} - SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_free_stack(struct sljit_stack *stack, void *allocator_data) { SLJIT_UNUSED_ARG(allocator_data); @@ -281,19 +218,7 @@ SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_free_stack(struct sljit_stack *st SLJIT_FREE(stack, allocator_data); } -#else /* ! defined _WIN32 */ - -SLJIT_INLINE static sljit_sw get_page_alignment(void) { - static sljit_sw sljit_page_align; - if (!sljit_page_align) { - sljit_page_align = sysconf(_SC_PAGESIZE); - /* Should never happen. */ - if (sljit_page_align < 0) - sljit_page_align = 4096; - sljit_page_align--; - } - return sljit_page_align; -} +#else /* !_WIN32 */ SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_free_stack(struct sljit_stack *stack, void *allocator_data) { @@ -302,7 +227,7 @@ SLJIT_API_FUNC_ATTRIBUTE void SLJIT_FUNC sljit_free_stack(struct sljit_stack *st SLJIT_FREE(stack, allocator_data); } -#endif /* defined _WIN32 */ +#endif /* _WIN32 */ SLJIT_API_FUNC_ATTRIBUTE struct sljit_stack* SLJIT_FUNC sljit_allocate_stack(sljit_uw start_size, sljit_uw max_size, void *allocator_data) { @@ -342,11 +267,9 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_stack* SLJIT_FUNC sljit_allocate_stack(slj #ifdef MAP_ANON ptr = mmap(NULL, max_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); #else /* !MAP_ANON */ - if (dev_zero < 0) { - if (open_dev_zero() != 0) { - SLJIT_FREE(stack, allocator_data); - return NULL; - } + if (SLJIT_UNLIKELY((dev_zero < 0) && open_dev_zero())) { + SLJIT_FREE(stack, allocator_data); + return NULL; } ptr = mmap(NULL, max_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, dev_zero, 0); #endif /* MAP_ANON */ @@ -365,7 +288,7 @@ SLJIT_API_FUNC_ATTRIBUTE struct sljit_stack* SLJIT_FUNC sljit_allocate_stack(slj SLJIT_API_FUNC_ATTRIBUTE sljit_u8 *SLJIT_FUNC sljit_stack_resize(struct sljit_stack *stack, sljit_u8 *new_start) { -#if defined _WIN32 || defined(MADV_DONTNEED) +#if defined _WIN32 || defined(POSIX_MADV_DONTNEED) sljit_uw aligned_old_start; sljit_uw aligned_new_start; sljit_sw page_align; @@ -389,15 +312,19 @@ SLJIT_API_FUNC_ATTRIBUTE sljit_u8 *SLJIT_FUNC sljit_stack_resize(struct sljit_st return NULL; } } -#elif defined(MADV_DONTNEED) +#elif defined(POSIX_MADV_DONTNEED) if (stack->start < new_start) { page_align = get_page_alignment(); aligned_new_start = (sljit_uw)new_start & ~page_align; aligned_old_start = ((sljit_uw)stack->start) & ~page_align; - /* If madvise is available, we release the unnecessary space. */ - if (aligned_new_start > aligned_old_start) - madvise((void*)aligned_old_start, aligned_new_start - aligned_old_start, MADV_DONTNEED); + + if (aligned_new_start > aligned_old_start) { + posix_madvise((void*)aligned_old_start, aligned_new_start - aligned_old_start, POSIX_MADV_DONTNEED); +#ifdef MADV_FREE + madvise((void*)aligned_old_start, aligned_new_start - aligned_old_start, MADV_FREE); +#endif /* MADV_FREE */ + } } #endif /* _WIN32 */ diff --git a/src/sljit/sljitWXExecAllocator.c b/src/sljit/sljitWXExecAllocator.c new file mode 100644 index 0000000..6ef71f7 --- /dev/null +++ b/src/sljit/sljitWXExecAllocator.c @@ -0,0 +1,225 @@ +/* + * Stack-less Just-In-Time compiler + * + * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are + * permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list + * of conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + This file contains a simple W^X executable memory allocator for POSIX + like systems and Windows + + In *NIX, MAP_ANON is required (that is considered a feature) so make + sure to set the right availability macros for your system or the code + will fail to build. + + If your system doesn't support mapping of anonymous pages (ex: IRIX) it + is also likely that it doesn't need this allocator and should be using + the standard one instead. + + It allocates a separate map for each code block and may waste a lot of + memory, because whatever was requested, will be rounded up to the page + size (minimum 4KB, but could be even bigger). + + It changes the page permissions (RW <-> RX) as needed and therefore, if you + will be updating the code after it has been generated, need to make sure to + block any concurrent execution, or could result in a SIGBUS, that could + even manifest itself at a different address than the one that was being + modified. + + Only use if you are unable to use the regular allocator because of security + restrictions and adding exceptions to your application or the system are + not possible. +*/ + +#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec) \ + sljit_update_wx_flags((from), (to), (enable_exec)) + +#ifndef _WIN32 +#include +#include + +#ifdef __NetBSD__ +#if defined(PROT_MPROTECT) +#define check_se_protected(ptr, size) (0) +#define SLJIT_PROT_WX PROT_MPROTECT(PROT_EXEC) +#else /* !PROT_MPROTECT */ +#ifdef _NETBSD_SOURCE +#include +#else /* !_NETBSD_SOURCE */ +typedef unsigned int u_int; +#define devmajor_t sljit_s32 +#endif /* _NETBSD_SOURCE */ +#include +#include + +#define check_se_protected(ptr, size) netbsd_se_protected() + +static SLJIT_INLINE int netbsd_se_protected(void) +{ + int mib[3]; + int paxflags; + size_t len = sizeof(paxflags); + + mib[0] = CTL_PROC; + mib[1] = getpid(); + mib[2] = PROC_PID_PAXFLAGS; + + if (SLJIT_UNLIKELY(sysctl(mib, 3, &paxflags, &len, NULL, 0) < 0)) + return -1; + + return (paxflags & CTL_PROC_PAXFLAGS_MPROTECT) ? -1 : 0; +} +#endif /* PROT_MPROTECT */ +#else /* POSIX */ +#define check_se_protected(ptr, size) generic_se_protected(ptr, size) + +static SLJIT_INLINE int generic_se_protected(void *ptr, sljit_uw size) +{ + if (SLJIT_LIKELY(!mprotect(ptr, size, PROT_EXEC))) + return mprotect(ptr, size, PROT_READ | PROT_WRITE); + + return -1; +} +#endif /* NetBSD */ + +#if defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED +#define SLJIT_SE_LOCK() +#define SLJIT_SE_UNLOCK() +#else /* !SLJIT_SINGLE_THREADED */ +#include +#define SLJIT_SE_LOCK() pthread_mutex_lock(&se_lock) +#define SLJIT_SE_UNLOCK() pthread_mutex_unlock(&se_lock) +#endif /* SLJIT_SINGLE_THREADED */ + +#ifndef SLJIT_PROT_WX +#define SLJIT_PROT_WX 0 +#endif /* !SLJIT_PROT_WX */ + +SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) +{ +#if !(defined SLJIT_SINGLE_THREADED && SLJIT_SINGLE_THREADED) + static pthread_mutex_t se_lock = PTHREAD_MUTEX_INITIALIZER; +#endif + static int se_protected = !SLJIT_PROT_WX; + sljit_uw* ptr; + + if (SLJIT_UNLIKELY(se_protected < 0)) + return NULL; + + size += sizeof(sljit_uw); + ptr = (sljit_uw*)mmap(NULL, size, PROT_READ | PROT_WRITE | SLJIT_PROT_WX, + MAP_PRIVATE | MAP_ANON, -1, 0); + + if (ptr == MAP_FAILED) + return NULL; + + if (SLJIT_UNLIKELY(se_protected > 0)) { + SLJIT_SE_LOCK(); + se_protected = check_se_protected(ptr, size); + SLJIT_SE_UNLOCK(); + if (SLJIT_UNLIKELY(se_protected < 0)) { + munmap((void *)ptr, size); + return NULL; + } + } + + *ptr++ = size; + return ptr; +} + +#undef SLJIT_PROT_WX +#undef SLJIT_SE_UNLOCK +#undef SLJIT_SE_LOCK + +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) +{ + sljit_uw *start_ptr = ((sljit_uw*)ptr) - 1; + munmap((void*)start_ptr, *start_ptr); +} + +static void sljit_update_wx_flags(void *from, void *to, sljit_s32 enable_exec) +{ + sljit_uw page_mask = (sljit_uw)get_page_alignment(); + sljit_uw start = (sljit_uw)from; + sljit_uw end = (sljit_uw)to; + int prot = PROT_READ | (enable_exec ? PROT_EXEC : PROT_WRITE); + + SLJIT_ASSERT(start < end); + + start &= ~page_mask; + end = (end + page_mask) & ~page_mask; + + mprotect((void*)start, end - start, prot); +} + +#else /* windows */ + +SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) +{ + sljit_uw *ptr; + + size += sizeof(sljit_uw); + ptr = (sljit_uw*)VirtualAlloc(NULL, size, + MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + + if (!ptr) + return NULL; + + *ptr++ = size; + + return ptr; +} + +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) +{ + sljit_uw start = (sljit_uw)ptr - sizeof(sljit_uw); +#if defined(SLJIT_DEBUG) && SLJIT_DEBUG + sljit_uw page_mask = (sljit_uw)get_page_alignment(); + + SLJIT_ASSERT(!(start & page_mask)); +#endif + VirtualFree((void*)start, 0, MEM_RELEASE); +} + +static void sljit_update_wx_flags(void *from, void *to, sljit_s32 enable_exec) +{ + DWORD oldprot; + sljit_uw page_mask = (sljit_uw)get_page_alignment(); + sljit_uw start = (sljit_uw)from; + sljit_uw end = (sljit_uw)to; + DWORD prot = enable_exec ? PAGE_EXECUTE : PAGE_READWRITE; + + SLJIT_ASSERT(start < end); + + start &= ~page_mask; + end = (end + page_mask) & ~page_mask; + + VirtualProtect((void*)start, end - start, prot, &oldprot); +} + +#endif /* !windows */ + +SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) +{ + /* This allocator does not keep unused memory for future allocations. */ +} -- cgit v1.2.1