diff options
Diffstat (limited to 'Modules/_ctypes/libffi/src/aarch64/ffi.c')
| -rw-r--r-- | Modules/_ctypes/libffi/src/aarch64/ffi.c | 1168 |
1 files changed, 0 insertions, 1168 deletions
diff --git a/Modules/_ctypes/libffi/src/aarch64/ffi.c b/Modules/_ctypes/libffi/src/aarch64/ffi.c deleted file mode 100644 index b807a2d381..0000000000 --- a/Modules/_ctypes/libffi/src/aarch64/ffi.c +++ /dev/null @@ -1,1168 +0,0 @@ -/* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -``Software''), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - -#include <stdio.h> - -#include <ffi.h> -#include <ffi_common.h> - -#include <stdlib.h> - -/* Stack alignment requirement in bytes */ -#if defined (__APPLE__) -#define AARCH64_STACK_ALIGN 1 -#else -#define AARCH64_STACK_ALIGN 16 -#endif - -#define N_X_ARG_REG 8 -#define N_V_ARG_REG 8 - -#define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT) - -union _d -{ - UINT64 d; - UINT32 s[2]; -}; - -struct call_context -{ - UINT64 x [AARCH64_N_XREG]; - struct - { - union _d d[2]; - } v [AARCH64_N_VREG]; -}; - -#if defined (__clang__) && defined (__APPLE__) -extern void -sys_icache_invalidate (void *start, size_t len); -#endif - -static inline void -ffi_clear_cache (void *start, void *end) -{ -#if defined (__clang__) && defined (__APPLE__) - sys_icache_invalidate (start, (char *)end - (char *)start); -#elif defined (__GNUC__) - __builtin___clear_cache (start, end); -#else -#error "Missing builtin to flush instruction cache" -#endif -} - -static void * -get_x_addr (struct call_context *context, unsigned n) -{ - return &context->x[n]; -} - -static void * -get_s_addr (struct call_context *context, unsigned n) -{ -#if defined __AARCH64EB__ - return &context->v[n].d[1].s[1]; -#else - return &context->v[n].d[0].s[0]; -#endif -} - -static void * -get_d_addr (struct call_context *context, unsigned n) -{ -#if defined __AARCH64EB__ - return &context->v[n].d[1]; -#else - return &context->v[n].d[0]; -#endif -} - -static void * -get_v_addr (struct call_context *context, unsigned n) -{ - return &context->v[n]; -} - -/* Return the memory location at which a basic type would reside - were it to have been stored in register n. */ - -static void * -get_basic_type_addr (unsigned short type, struct call_context *context, - unsigned n) -{ - switch (type) - { - case FFI_TYPE_FLOAT: - return get_s_addr (context, n); - case FFI_TYPE_DOUBLE: - return get_d_addr (context, n); -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: - return get_v_addr (context, n); -#endif - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_INT: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - return get_x_addr (context, n); - case FFI_TYPE_VOID: - return NULL; - default: - FFI_ASSERT (0); - return NULL; - } -} - -/* Return the alignment width for each of the basic types. */ - -static size_t -get_basic_type_alignment (unsigned short type) -{ - switch (type) - { - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: - return sizeof (UINT64); -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: - return sizeof (long double); -#endif - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: -#if defined (__APPLE__) - return sizeof (UINT8); -#endif - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: -#if defined (__APPLE__) - return sizeof (UINT16); -#endif - case FFI_TYPE_UINT32: - case FFI_TYPE_INT: - case FFI_TYPE_SINT32: -#if defined (__APPLE__) - return sizeof (UINT32); -#endif - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - return sizeof (UINT64); - - default: - FFI_ASSERT (0); - return 0; - } -} - -/* Return the size in bytes for each of the basic types. */ - -static size_t -get_basic_type_size (unsigned short type) -{ - switch (type) - { - case FFI_TYPE_FLOAT: - return sizeof (UINT32); - case FFI_TYPE_DOUBLE: - return sizeof (UINT64); -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: - return sizeof (long double); -#endif - case FFI_TYPE_UINT8: - return sizeof (UINT8); - case FFI_TYPE_SINT8: - return sizeof (SINT8); - case FFI_TYPE_UINT16: - return sizeof (UINT16); - case FFI_TYPE_SINT16: - return sizeof (SINT16); - case FFI_TYPE_UINT32: - return sizeof (UINT32); - case FFI_TYPE_INT: - case FFI_TYPE_SINT32: - return sizeof (SINT32); - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - return sizeof (UINT64); - case FFI_TYPE_SINT64: - return sizeof (SINT64); - - default: - FFI_ASSERT (0); - return 0; - } -} - -extern void -ffi_call_SYSV (unsigned (*)(struct call_context *context, unsigned char *, - extended_cif *), - struct call_context *context, - extended_cif *, - size_t, - void (*fn)(void)); - -extern void -ffi_closure_SYSV (ffi_closure *); - -/* Test for an FFI floating point representation. */ - -static unsigned -is_floating_type (unsigned short type) -{ - return (type == FFI_TYPE_FLOAT || type == FFI_TYPE_DOUBLE - || type == FFI_TYPE_LONGDOUBLE); -} - -/* Test for a homogeneous structure. */ - -static unsigned short -get_homogeneous_type (ffi_type *ty) -{ - if (ty->type == FFI_TYPE_STRUCT && ty->elements) - { - unsigned i; - unsigned short candidate_type - = get_homogeneous_type (ty->elements[0]); - for (i =1; ty->elements[i]; i++) - { - unsigned short iteration_type = 0; - /* If we have a nested struct, we must find its homogeneous type. - If that fits with our candidate type, we are still - homogeneous. */ - if (ty->elements[i]->type == FFI_TYPE_STRUCT - && ty->elements[i]->elements) - { - iteration_type = get_homogeneous_type (ty->elements[i]); - } - else - { - iteration_type = ty->elements[i]->type; - } - - /* If we are not homogeneous, return FFI_TYPE_STRUCT. */ - if (candidate_type != iteration_type) - return FFI_TYPE_STRUCT; - } - return candidate_type; - } - - /* Base case, we have no more levels of nesting, so we - are a basic type, and so, trivially homogeneous in that type. */ - return ty->type; -} - -/* Determine the number of elements within a STRUCT. - - Note, we must handle nested structs. - - If ty is not a STRUCT this function will return 0. */ - -static unsigned -element_count (ffi_type *ty) -{ - if (ty->type == FFI_TYPE_STRUCT && ty->elements) - { - unsigned n; - unsigned elems = 0; - for (n = 0; ty->elements[n]; n++) - { - if (ty->elements[n]->type == FFI_TYPE_STRUCT - && ty->elements[n]->elements) - elems += element_count (ty->elements[n]); - else - elems++; - } - return elems; - } - return 0; -} - -/* Test for a homogeneous floating point aggregate. - - A homogeneous floating point aggregate is a homogeneous aggregate of - a half- single- or double- precision floating point type with one - to four elements. Note that this includes nested structs of the - basic type. */ - -static int -is_hfa (ffi_type *ty) -{ - if (ty->type == FFI_TYPE_STRUCT - && ty->elements[0] - && is_floating_type (get_homogeneous_type (ty))) - { - unsigned n = element_count (ty); - return n >= 1 && n <= 4; - } - return 0; -} - -/* Test if an ffi_type is a candidate for passing in a register. - - This test does not check that sufficient registers of the - appropriate class are actually available, merely that IFF - sufficient registers are available then the argument will be passed - in register(s). - - Note that an ffi_type that is deemed to be a register candidate - will always be returned in registers. - - Returns 1 if a register candidate else 0. */ - -static int -is_register_candidate (ffi_type *ty) -{ - switch (ty->type) - { - case FFI_TYPE_VOID: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: -#endif - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_UINT64: - case FFI_TYPE_POINTER: - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_SINT32: - case FFI_TYPE_INT: - case FFI_TYPE_SINT64: - return 1; - - case FFI_TYPE_STRUCT: - if (is_hfa (ty)) - { - return 1; - } - else if (ty->size > 16) - { - /* Too large. Will be replaced with a pointer to memory. The - pointer MAY be passed in a register, but the value will - not. This test specifically fails since the argument will - never be passed by value in registers. */ - return 0; - } - else - { - /* Might be passed in registers depending on the number of - registers required. */ - return (ty->size + 7) / 8 < N_X_ARG_REG; - } - break; - - default: - FFI_ASSERT (0); - break; - } - - return 0; -} - -/* Test if an ffi_type argument or result is a candidate for a vector - register. */ - -static int -is_v_register_candidate (ffi_type *ty) -{ - return is_floating_type (ty->type) - || (ty->type == FFI_TYPE_STRUCT && is_hfa (ty)); -} - -/* Representation of the procedure call argument marshalling - state. - - The terse state variable names match the names used in the AARCH64 - PCS. */ - -struct arg_state -{ - unsigned ngrn; /* Next general-purpose register number. */ - unsigned nsrn; /* Next vector register number. */ - size_t nsaa; /* Next stack offset. */ - -#if defined (__APPLE__) - unsigned allocating_variadic; -#endif -}; - -/* Initialize a procedure call argument marshalling state. */ -static void -arg_init (struct arg_state *state, size_t call_frame_size) -{ - state->ngrn = 0; - state->nsrn = 0; - state->nsaa = 0; - -#if defined (__APPLE__) - state->allocating_variadic = 0; -#endif -} - -/* Return the number of available consecutive core argument - registers. */ - -static unsigned -available_x (struct arg_state *state) -{ - return N_X_ARG_REG - state->ngrn; -} - -/* Return the number of available consecutive vector argument - registers. */ - -static unsigned -available_v (struct arg_state *state) -{ - return N_V_ARG_REG - state->nsrn; -} - -static void * -allocate_to_x (struct call_context *context, struct arg_state *state) -{ - FFI_ASSERT (state->ngrn < N_X_ARG_REG); - return get_x_addr (context, (state->ngrn)++); -} - -static void * -allocate_to_s (struct call_context *context, struct arg_state *state) -{ - FFI_ASSERT (state->nsrn < N_V_ARG_REG); - return get_s_addr (context, (state->nsrn)++); -} - -static void * -allocate_to_d (struct call_context *context, struct arg_state *state) -{ - FFI_ASSERT (state->nsrn < N_V_ARG_REG); - return get_d_addr (context, (state->nsrn)++); -} - -static void * -allocate_to_v (struct call_context *context, struct arg_state *state) -{ - FFI_ASSERT (state->nsrn < N_V_ARG_REG); - return get_v_addr (context, (state->nsrn)++); -} - -/* Allocate an aligned slot on the stack and return a pointer to it. */ -static void * -allocate_to_stack (struct arg_state *state, void *stack, size_t alignment, - size_t size) -{ - void *allocation; - - /* Round up the NSAA to the larger of 8 or the natural - alignment of the argument's type. */ - state->nsaa = ALIGN (state->nsaa, alignment); - state->nsaa = ALIGN (state->nsaa, alignment); -#if defined (__APPLE__) - if (state->allocating_variadic) - state->nsaa = ALIGN (state->nsaa, 8); -#else - state->nsaa = ALIGN (state->nsaa, 8); -#endif - - allocation = stack + state->nsaa; - - state->nsaa += size; - return allocation; -} - -static void -copy_basic_type (void *dest, void *source, unsigned short type) -{ - /* This is necessary to ensure that basic types are copied - sign extended to 64-bits as libffi expects. */ - switch (type) - { - case FFI_TYPE_FLOAT: - *(float *) dest = *(float *) source; - break; - case FFI_TYPE_DOUBLE: - *(double *) dest = *(double *) source; - break; -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: - *(long double *) dest = *(long double *) source; - break; -#endif - case FFI_TYPE_UINT8: - *(ffi_arg *) dest = *(UINT8 *) source; - break; - case FFI_TYPE_SINT8: - *(ffi_sarg *) dest = *(SINT8 *) source; - break; - case FFI_TYPE_UINT16: - *(ffi_arg *) dest = *(UINT16 *) source; - break; - case FFI_TYPE_SINT16: - *(ffi_sarg *) dest = *(SINT16 *) source; - break; - case FFI_TYPE_UINT32: - *(ffi_arg *) dest = *(UINT32 *) source; - break; - case FFI_TYPE_INT: - case FFI_TYPE_SINT32: - *(ffi_sarg *) dest = *(SINT32 *) source; - break; - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - *(ffi_arg *) dest = *(UINT64 *) source; - break; - case FFI_TYPE_SINT64: - *(ffi_sarg *) dest = *(SINT64 *) source; - break; - case FFI_TYPE_VOID: - break; - - default: - FFI_ASSERT (0); - } -} - -static void -copy_hfa_to_reg_or_stack (void *memory, - ffi_type *ty, - struct call_context *context, - unsigned char *stack, - struct arg_state *state) -{ - unsigned elems = element_count (ty); - if (available_v (state) < elems) - { - /* There are insufficient V registers. Further V register allocations - are prevented, the NSAA is adjusted (by allocate_to_stack ()) - and the argument is copied to memory at the adjusted NSAA. */ - state->nsrn = N_V_ARG_REG; - memcpy (allocate_to_stack (state, stack, ty->alignment, ty->size), - memory, - ty->size); - } - else - { - int i; - unsigned short type = get_homogeneous_type (ty); - for (i = 0; i < elems; i++) - { - void *reg = allocate_to_v (context, state); - copy_basic_type (reg, memory, type); - memory += get_basic_type_size (type); - } - } -} - -/* Either allocate an appropriate register for the argument type, or if - none are available, allocate a stack slot and return a pointer - to the allocated space. */ - -static void * -allocate_to_register_or_stack (struct call_context *context, - unsigned char *stack, - struct arg_state *state, - unsigned short type) -{ - size_t alignment = get_basic_type_alignment (type); - size_t size = alignment; - switch (type) - { - case FFI_TYPE_FLOAT: - /* This is the only case for which the allocated stack size - should not match the alignment of the type. */ - size = sizeof (UINT32); - /* Fall through. */ - case FFI_TYPE_DOUBLE: - if (state->nsrn < N_V_ARG_REG) - return allocate_to_d (context, state); - state->nsrn = N_V_ARG_REG; - break; -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: - if (state->nsrn < N_V_ARG_REG) - return allocate_to_v (context, state); - state->nsrn = N_V_ARG_REG; - break; -#endif - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_INT: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - if (state->ngrn < N_X_ARG_REG) - return allocate_to_x (context, state); - state->ngrn = N_X_ARG_REG; - break; - default: - FFI_ASSERT (0); - } - - return allocate_to_stack (state, stack, alignment, size); -} - -/* Copy a value to an appropriate register, or if none are - available, to the stack. */ - -static void -copy_to_register_or_stack (struct call_context *context, - unsigned char *stack, - struct arg_state *state, - void *value, - unsigned short type) -{ - copy_basic_type ( - allocate_to_register_or_stack (context, stack, state, type), - value, - type); -} - -/* Marshall the arguments from FFI representation to procedure call - context and stack. */ - -static unsigned -aarch64_prep_args (struct call_context *context, unsigned char *stack, - extended_cif *ecif) -{ - int i; - struct arg_state state; - - arg_init (&state, ALIGN(ecif->cif->bytes, 16)); - - for (i = 0; i < ecif->cif->nargs; i++) - { - ffi_type *ty = ecif->cif->arg_types[i]; - switch (ty->type) - { - case FFI_TYPE_VOID: - FFI_ASSERT (0); - break; - - /* If the argument is a basic type the argument is allocated to an - appropriate register, or if none are available, to the stack. */ - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: -#endif - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_INT: - case FFI_TYPE_SINT32: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - copy_to_register_or_stack (context, stack, &state, - ecif->avalue[i], ty->type); - break; - - case FFI_TYPE_STRUCT: - if (is_hfa (ty)) - { - copy_hfa_to_reg_or_stack (ecif->avalue[i], ty, context, - stack, &state); - } - else if (ty->size > 16) - { - /* If the argument is a composite type that is larger than 16 - bytes, then the argument has been copied to memory, and - the argument is replaced by a pointer to the copy. */ - - copy_to_register_or_stack (context, stack, &state, - &(ecif->avalue[i]), FFI_TYPE_POINTER); - } - else if (available_x (&state) >= (ty->size + 7) / 8) - { - /* If the argument is a composite type and the size in - double-words is not more than the number of available - X registers, then the argument is copied into consecutive - X registers. */ - int j; - for (j = 0; j < (ty->size + 7) / 8; j++) - { - memcpy (allocate_to_x (context, &state), - &(((UINT64 *) ecif->avalue[i])[j]), - sizeof (UINT64)); - } - } - else - { - /* Otherwise, there are insufficient X registers. Further X - register allocations are prevented, the NSAA is adjusted - (by allocate_to_stack ()) and the argument is copied to - memory at the adjusted NSAA. */ - state.ngrn = N_X_ARG_REG; - - memcpy (allocate_to_stack (&state, stack, ty->alignment, - ty->size), ecif->avalue + i, ty->size); - } - break; - - default: - FFI_ASSERT (0); - break; - } - -#if defined (__APPLE__) - if (i + 1 == ecif->cif->aarch64_nfixedargs) - { - state.ngrn = N_X_ARG_REG; - state.nsrn = N_V_ARG_REG; - - state.allocating_variadic = 1; - } -#endif - } - - return ecif->cif->aarch64_flags; -} - -ffi_status -ffi_prep_cif_machdep (ffi_cif *cif) -{ - /* Round the stack up to a multiple of the stack alignment requirement. */ - cif->bytes = - (cif->bytes + (AARCH64_STACK_ALIGN - 1)) & ~ (AARCH64_STACK_ALIGN - 1); - - /* Initialize our flags. We are interested if this CIF will touch a - vector register, if so we will enable context save and load to - those registers, otherwise not. This is intended to be friendly - to lazy float context switching in the kernel. */ - cif->aarch64_flags = 0; - - if (is_v_register_candidate (cif->rtype)) - { - cif->aarch64_flags |= AARCH64_FFI_WITH_V; - } - else - { - int i; - for (i = 0; i < cif->nargs; i++) - if (is_v_register_candidate (cif->arg_types[i])) - { - cif->aarch64_flags |= AARCH64_FFI_WITH_V; - break; - } - } - - return FFI_OK; -} - -#if defined (__APPLE__) - -/* Perform Apple-specific cif processing for variadic calls */ -ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, - unsigned int nfixedargs, - unsigned int ntotalargs) -{ - cif->aarch64_nfixedargs = nfixedargs; - - return ffi_prep_cif_machdep(cif); -} - -#endif - -/* Call a function with the provided arguments and capture the return - value. */ -void -ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue) -{ - extended_cif ecif; - - ecif.cif = cif; - ecif.avalue = avalue; - ecif.rvalue = rvalue; - - switch (cif->abi) - { - case FFI_SYSV: - { - struct call_context context; - size_t stack_bytes; - - /* Figure out the total amount of stack space we need, the - above call frame space needs to be 16 bytes aligned to - ensure correct alignment of the first object inserted in - that space hence the ALIGN applied to cif->bytes.*/ - stack_bytes = ALIGN(cif->bytes, 16); - - memset (&context, 0, sizeof (context)); - if (is_register_candidate (cif->rtype)) - { - ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn); - switch (cif->rtype->type) - { - case FFI_TYPE_VOID: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: -#endif - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_INT: - case FFI_TYPE_SINT64: - { - void *addr = get_basic_type_addr (cif->rtype->type, - &context, 0); - copy_basic_type (rvalue, addr, cif->rtype->type); - break; - } - - case FFI_TYPE_STRUCT: - if (is_hfa (cif->rtype)) - { - int j; - unsigned short type = get_homogeneous_type (cif->rtype); - unsigned elems = element_count (cif->rtype); - for (j = 0; j < elems; j++) - { - void *reg = get_basic_type_addr (type, &context, j); - copy_basic_type (rvalue, reg, type); - rvalue += get_basic_type_size (type); - } - } - else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG) - { - size_t size = ALIGN (cif->rtype->size, sizeof (UINT64)); - memcpy (rvalue, get_x_addr (&context, 0), size); - } - else - { - FFI_ASSERT (0); - } - break; - - default: - FFI_ASSERT (0); - break; - } - } - else - { - memcpy (get_x_addr (&context, 8), &rvalue, sizeof (UINT64)); - ffi_call_SYSV (aarch64_prep_args, &context, &ecif, - stack_bytes, fn); - } - break; - } - - default: - FFI_ASSERT (0); - break; - } -} - -static unsigned char trampoline [] = -{ 0x70, 0x00, 0x00, 0x58, /* ldr x16, 1f */ - 0x91, 0x00, 0x00, 0x10, /* adr x17, 2f */ - 0x00, 0x02, 0x1f, 0xd6 /* br x16 */ -}; - -/* Build a trampoline. */ - -#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS) \ - ({unsigned char *__tramp = (unsigned char*)(TRAMP); \ - UINT64 __fun = (UINT64)(FUN); \ - UINT64 __ctx = (UINT64)(CTX); \ - UINT64 __flags = (UINT64)(FLAGS); \ - memcpy (__tramp, trampoline, sizeof (trampoline)); \ - memcpy (__tramp + 12, &__fun, sizeof (__fun)); \ - memcpy (__tramp + 20, &__ctx, sizeof (__ctx)); \ - memcpy (__tramp + 28, &__flags, sizeof (__flags)); \ - ffi_clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE); \ - }) - -ffi_status -ffi_prep_closure_loc (ffi_closure* closure, - ffi_cif* cif, - void (*fun)(ffi_cif*,void*,void**,void*), - void *user_data, - void *codeloc) -{ - if (cif->abi != FFI_SYSV) - return FFI_BAD_ABI; - - FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_SYSV, codeloc, - cif->aarch64_flags); - - closure->cif = cif; - closure->user_data = user_data; - closure->fun = fun; - - return FFI_OK; -} - -/* Primary handler to setup and invoke a function within a closure. - - A closure when invoked enters via the assembler wrapper - ffi_closure_SYSV(). The wrapper allocates a call context on the - stack, saves the interesting registers (from the perspective of - the calling convention) into the context then passes control to - ffi_closure_SYSV_inner() passing the saved context and a pointer to - the stack at the point ffi_closure_SYSV() was invoked. - - On the return path the assembler wrapper will reload call context - registers. - - ffi_closure_SYSV_inner() marshalls the call context into ffi value - descriptors, invokes the wrapped function, then marshalls the return - value back into the call context. */ - -void FFI_HIDDEN -ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context, - void *stack) -{ - ffi_cif *cif = closure->cif; - void **avalue = (void**) alloca (cif->nargs * sizeof (void*)); - void *rvalue = NULL; - int i; - struct arg_state state; - - arg_init (&state, ALIGN(cif->bytes, 16)); - - for (i = 0; i < cif->nargs; i++) - { - ffi_type *ty = cif->arg_types[i]; - - switch (ty->type) - { - case FFI_TYPE_VOID: - FFI_ASSERT (0); - break; - - case FFI_TYPE_UINT8: - case FFI_TYPE_SINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_SINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_SINT32: - case FFI_TYPE_INT: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT64: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: - avalue[i] = allocate_to_register_or_stack (context, stack, - &state, ty->type); - break; -#endif - - case FFI_TYPE_STRUCT: - if (is_hfa (ty)) - { - unsigned n = element_count (ty); - if (available_v (&state) < n) - { - state.nsrn = N_V_ARG_REG; - avalue[i] = allocate_to_stack (&state, stack, ty->alignment, - ty->size); - } - else - { - switch (get_homogeneous_type (ty)) - { - case FFI_TYPE_FLOAT: - { - /* Eeek! We need a pointer to the structure, - however the homogeneous float elements are - being passed in individual S registers, - therefore the structure is not represented as - a contiguous sequence of bytes in our saved - register context. We need to fake up a copy - of the structure laid out in memory - correctly. The fake can be tossed once the - closure function has returned hence alloca() - is sufficient. */ - int j; - UINT32 *p = avalue[i] = alloca (ty->size); - for (j = 0; j < element_count (ty); j++) - memcpy (&p[j], - allocate_to_s (context, &state), - sizeof (*p)); - break; - } - - case FFI_TYPE_DOUBLE: - { - /* Eeek! We need a pointer to the structure, - however the homogeneous float elements are - being passed in individual S registers, - therefore the structure is not represented as - a contiguous sequence of bytes in our saved - register context. We need to fake up a copy - of the structure laid out in memory - correctly. The fake can be tossed once the - closure function has returned hence alloca() - is sufficient. */ - int j; - UINT64 *p = avalue[i] = alloca (ty->size); - for (j = 0; j < element_count (ty); j++) - memcpy (&p[j], - allocate_to_d (context, &state), - sizeof (*p)); - break; - } - -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: - memcpy (&avalue[i], - allocate_to_v (context, &state), - sizeof (*avalue)); - break; -#endif - - default: - FFI_ASSERT (0); - break; - } - } - } - else if (ty->size > 16) - { - /* Replace Composite type of size greater than 16 with a - pointer. */ - memcpy (&avalue[i], - allocate_to_register_or_stack (context, stack, - &state, FFI_TYPE_POINTER), - sizeof (avalue[i])); - } - else if (available_x (&state) >= (ty->size + 7) / 8) - { - avalue[i] = get_x_addr (context, state.ngrn); - state.ngrn += (ty->size + 7) / 8; - } - else - { - state.ngrn = N_X_ARG_REG; - - avalue[i] = allocate_to_stack (&state, stack, ty->alignment, - ty->size); - } - break; - - default: - FFI_ASSERT (0); - break; - } - } - - /* Figure out where the return value will be passed, either in - registers or in a memory block allocated by the caller and passed - in x8. */ - - if (is_register_candidate (cif->rtype)) - { - /* Register candidates are *always* returned in registers. */ - - /* Allocate a scratchpad for the return value, we will let the - callee scrible the result into the scratch pad then move the - contents into the appropriate return value location for the - call convention. */ - rvalue = alloca (cif->rtype->size); - (closure->fun) (cif, rvalue, avalue, closure->user_data); - - /* Copy the return value into the call context so that it is returned - as expected to our caller. */ - switch (cif->rtype->type) - { - case FFI_TYPE_VOID: - break; - - case FFI_TYPE_UINT8: - case FFI_TYPE_UINT16: - case FFI_TYPE_UINT32: - case FFI_TYPE_POINTER: - case FFI_TYPE_UINT64: - case FFI_TYPE_SINT8: - case FFI_TYPE_SINT16: - case FFI_TYPE_INT: - case FFI_TYPE_SINT32: - case FFI_TYPE_SINT64: - case FFI_TYPE_FLOAT: - case FFI_TYPE_DOUBLE: -#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE - case FFI_TYPE_LONGDOUBLE: -#endif - { - void *addr = get_basic_type_addr (cif->rtype->type, context, 0); - copy_basic_type (addr, rvalue, cif->rtype->type); - break; - } - case FFI_TYPE_STRUCT: - if (is_hfa (cif->rtype)) - { - int j; - unsigned short type = get_homogeneous_type (cif->rtype); - unsigned elems = element_count (cif->rtype); - for (j = 0; j < elems; j++) - { - void *reg = get_basic_type_addr (type, context, j); - copy_basic_type (reg, rvalue, type); - rvalue += get_basic_type_size (type); - } - } - else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG) - { - size_t size = ALIGN (cif->rtype->size, sizeof (UINT64)) ; - memcpy (get_x_addr (context, 0), rvalue, size); - } - else - { - FFI_ASSERT (0); - } - break; - default: - FFI_ASSERT (0); - break; - } - } - else - { - memcpy (&rvalue, get_x_addr (context, 8), sizeof (UINT64)); - (closure->fun) (cif, rvalue, avalue, closure->user_data); - } -} - |