diff options
| -rw-r--r-- | elf/Makefile | 2 | ||||
| -rw-r--r-- | elf/dl-ifunc.c | 192 | ||||
| -rw-r--r-- | elf/dl-ifunc.h | 88 | ||||
| -rw-r--r-- | elf/dl-open.c | 4 | ||||
| -rw-r--r-- | elf/rtld.c | 6 | ||||
| -rw-r--r-- | include/link.h | 6 | ||||
| -rw-r--r-- | sysdeps/x86_64/dl-ifunc-reloc.h | 91 | ||||
| -rw-r--r-- | sysdeps/x86_64/dl-machine.h | 26 |
8 files changed, 393 insertions, 22 deletions
diff --git a/elf/Makefile b/elf/Makefile index c7a29693bd..04c0013380 100644 --- a/elf/Makefile +++ b/elf/Makefile @@ -29,7 +29,7 @@ routines = $(all-dl-routines) dl-support dl-iteratephdr \ # The core dynamic linking functions are in libc for the static and # profiled libraries. dl-routines = $(addprefix dl-,load lookup object reloc deps hwcaps \ - runtime init fini debug misc \ + runtime init fini debug misc ifunc \ version profile conflict tls origin scope \ execstack caller open close trampoline) ifeq (yes,$(use-ldconfig)) diff --git a/elf/dl-ifunc.c b/elf/dl-ifunc.c new file mode 100644 index 0000000000..96e4d077d8 --- /dev/null +++ b/elf/dl-ifunc.c @@ -0,0 +1,192 @@ +/* Delayed IFUNC processing. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifdef HAVE_IFUNC + +# include <dl-ifunc.h> +# include <errno.h> +# include <ldsodefs.h> +# include <sys/mman.h> +# include <unistd.h> + +/* Machine-specific definitions. */ +# include <dl-ifunc-reloc.h> + +/* This struct covers a whole page containing individual struct + dl_ifunc_relocation elements, which are allocated individually by + allocate_relocation below. */ +struct dl_ifunc_relocation_array +{ + struct dl_ifunc_relocation_array *next; + struct dl_ifunc_relocation data[]; +}; + +/* Global list of allocated arrays of struct dl_ifunc_relocation + elements. */ +static struct dl_ifunc_relocation_array *array_list; + +/* Position of the next allocation in the current array used as the + source of struct dl_ifunc_relocation allocations. */ +static struct dl_ifunc_relocation *next_alloc; +static unsigned int remaining_alloc; + +/* Allocate a new struct dl_ifunc_relocation_array object. Return the + first data object therein. Update array_list, next_alloc, + remaining_alloc. This function assumes taht remaining_alloc is + zero. */ +static struct dl_ifunc_relocation * +allocate_array (void) +{ + size_t page_size = GLRO(dl_pagesize); + void *ptr = __mmap (NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED) + _dl_signal_error (ENOMEM, NULL, NULL, + "cannot allocate IFUNC resolver information"); + struct dl_ifunc_relocation_array *new_head = ptr; + new_head->next = array_list; + array_list = new_head; + next_alloc = &new_head->data[1]; + /* The function returns the first element of the new array. */ + remaining_alloc + = (page_size - offsetof (struct dl_ifunc_relocation_array, data)) + / sizeof (new_head->data[0]) - 1; + return &new_head->data[0]; +} + +/* Allocate whone struct dl_ifunc_relocation element from the active + array. Updated next_alloc, remaining_alloc, and potentially + array_list. */ +static struct dl_ifunc_relocation * +allocate_ifunc (void) +{ + if (remaining_alloc == 0) + return allocate_array (); + --remaining_alloc; + return next_alloc++; +} + +/* Deallocate the list of array allocations starting at + array_list. */ +static void +free_allocations (void) +{ + size_t page_size = GLRO(dl_pagesize); + struct dl_ifunc_relocation_array *p = array_list; + while (p != NULL) + { + struct dl_ifunc_relocation_array *next = p->next; + munmap (p, page_size); + p = next; + } + /* Restart from scratch if _dl_ifunc_record_reloc is called + again. */ + array_list = NULL; + next_alloc = NULL; + remaining_alloc = 0; +} + +/* Process all delayed IFUNC resolutions for IFUNC_MAP alone. */ +static void +apply_relocations (struct link_map *ifunc_map) +{ + if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS) + && ifunc_map->l_name != NULL) + _dl_debug_printf ("applying delayed IFUNC relocations against %s\n", + ifunc_map->l_name); + + struct dl_ifunc_relocation *p = ifunc_map->l_ifunc_relocations; + unsigned int count = 0; + while (p != NULL) + { + if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS) + && ifunc_map->l_name != NULL) + { + if (p->ifunc_sym == NULL) + _dl_debug_printf ("processing relative IFUNC relocation for %s\n", + ifunc_map->l_name); + else + { + const char *strtab = (const char *) D_PTR (ifunc_map, l_info[DT_STRTAB]); + _dl_debug_printf ("binding IFUNC symbol %s in %s against %s\n", + strtab + p->ifunc_sym->st_name, + p->reloc_map->l_name, ifunc_map->l_name); + } + } + _dl_ifunc_process_relocation (p, ifunc_map); + p = p->next; + ++count; + } + + if (__glibc_unlikely (GLRO(dl_debug_mask) & DL_DEBUG_BINDINGS) + && ifunc_map->l_name != NULL) + _dl_debug_printf ("%u delayed IFUNC relocations performed against %s\n", + count, ifunc_map->l_name); + + /* No deallocation takes place here. All allocated objects are + freed in bulk by a later call to free_allocations. */ + ifunc_map->l_ifunc_relocations = NULL; +} + +void internal_function +_dl_ifunc_record_reloc (struct link_map *reloc_map, + const ElfW(Rela) *reloc, + ElfW(Addr) *reloc_addr, + struct link_map *ifunc_map, + const ElfW(Sym) *ifunc_sym) +{ + /* Add the delayed IFUNC relocation to the list for ifunc_map. */ + struct dl_ifunc_relocation *ifunc = allocate_ifunc (); + *ifunc = (struct dl_ifunc_relocation) + { + .reloc_map = reloc_map, + .reloc = reloc, + .reloc_addr = reloc_addr, + .ifunc_sym = ifunc_sym, + .next = ifunc_map->l_ifunc_relocations, + }; + ifunc_map->l_ifunc_relocations = ifunc; +} + +void internal_function +_dl_ifunc_apply_relocations (struct link_map *new) +{ + /* Traverse the search list in reverse order, in case an IFUNC + resolver calls into one its dependencies, and those dependency + needs IFUNC resolution as well. */ + struct link_map **r_list = new->l_searchlist.r_list; + for (unsigned int i = new->l_searchlist.r_nlist; i-- > 0; ) + { + struct link_map *l = r_list[i]; + if (l->l_ifunc_relocations != NULL) + apply_relocations (l); + } + free_allocations (); +} + +void internal_function +_dl_ifunc_clear_relocations (struct link_map *map) +{ + Lmid_t nsid = map->l_ns; + struct link_namespaces *ns = &GL(dl_ns)[nsid]; + for (struct link_map *l = ns->_ns_loaded; l != NULL; l = l->l_next) + l->l_ifunc_relocations = NULL; + free_allocations (); +} + +#endif /* HAVE_IFUNC */ diff --git a/elf/dl-ifunc.h b/elf/dl-ifunc.h new file mode 100644 index 0000000000..39a5c12c73 --- /dev/null +++ b/elf/dl-ifunc.h @@ -0,0 +1,88 @@ +/* Private declarations for delayed IFUNC processing. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef DL_IFUNC_H +#define DL_IFUNC_H + +#include <link.h> + +/* All functions declared in this file must be called while the global + rtld lock is acquired. */ + +/* Apply all pending IFUNC relocations in the search scope of the new + link map. Deallocate all auxiliary allocations. */ +void _dl_ifunc_apply_relocations (struct link_map *new) + attribute_hidden internal_function; + +/* Clear all allocated delayed IFUNC relocations in the namespace of + the link map. Deallocate all auxiliary allocations. This function + is intended for clearing up after a dlopen failure. */ +void _dl_ifunc_clear_relocations (struct link_map *map) + attribute_hidden internal_function; + +/* The no-op implementatoin is only available if IFUNCs are + supported. */ +#ifdef HAVE_IFUNC + +/* Delayed IFUNC relocation. For each link map with a referenced + IFUNC symbol, a separate single-linked list of delayed IFUNC + relocations exists while the dynamic linker is running. */ +struct dl_ifunc_relocation +{ + /* Information about the relocation. */ + struct link_map *reloc_map; + const ElfW(Rela) *reloc; + ElfW(Addr) *reloc_addr; + + /* Information about the IFUNC resolver. The corresponding symbol + map is implied. */ + const ElfW(Sym) *ifunc_sym; + + /* Pointer to the next element in the list of IFUNC relocations for + the symbol map. */ + struct dl_ifunc_relocation *next; +}; + +/* Record a delayed IFUNC relocation for IFUNC_SYM at *RELOC_ADDR. + The relocation is associated with IFUNC_MAP. Can raise an error + using _dl_signal_error. IFUNC_SYM can be NULL if the relocation is + a relative IFUNC relocation. */ +void _dl_ifunc_record_reloc (struct link_map *reloc_map, + const ElfW(Rela) *reloc, + ElfW(Addr) *reloc_addr, + struct link_map *ifunc_map, + const ElfW(Sym) *ifunc_sym) + attribute_hidden internal_function; + +#else /* !HAVE_IFUNC */ + +/* Dummy implementations for targets without IFUNC support. */ + +static inline void +_dl_ifunc_apply_relocations (struct link_map *new) +{ +} + +static inline void +_dl_ifunc_clear_relocations (struct link_map *map) +{ +} + +#endif /* !HAVE_IFUNC */ + +#endif /* DL_IFUNC_H */ diff --git a/elf/dl-open.c b/elf/dl-open.c index d238fb2cc5..e40fb3b7e8 100644 --- a/elf/dl-open.c +++ b/elf/dl-open.c @@ -35,7 +35,7 @@ #include <atomic.h> #include <dl-dst.h> - +#include <dl-ifunc.h> extern int __libc_multiple_libcs; /* Defined in init-first.c. */ @@ -552,6 +552,7 @@ TLS generation counter wrapped! Please report this.")); } } + _dl_ifunc_apply_relocations (new); _dl_relocate_apply_relro (new); /* Notify the debugger all new objects have been relocated. */ @@ -673,6 +674,7 @@ no more namespaces available for dlmopen()")); if ((mode & __RTLD_AUDIT) == 0) GL(dl_tls_dtv_gaps) = true; + _dl_ifunc_clear_relocations (args.map); _dl_close_worker (args.map, true); } diff --git a/elf/rtld.c b/elf/rtld.c index 16119e8c31..cb9d330d66 100644 --- a/elf/rtld.c +++ b/elf/rtld.c @@ -41,6 +41,7 @@ #include <tls.h> #include <stap-probe.h> #include <stackinfo.h> +#include <dl-ifunc.h> #include <assert.h> @@ -2114,10 +2115,11 @@ ERROR: ld.so: object '%s' cannot be loaded as audit interface: %s; ignored.\n", HP_TIMING_ACCUM_NT (relocate_time, add); } - /* Activate RELRO protection. In the prelink case, this was already - done earlier. */ + /* Perform delayed IFUNC relocations and activate RELRO protection. + In the prelink case, this was already done earlier. */ if (! prelinked) { + _dl_ifunc_apply_relocations (main_map); /* Make sure that this covers the dynamic linker as well. TODO: rtld_multiple_ref is always true because libc.so needs the dynamic linker internally. */ diff --git a/include/link.h b/include/link.h index eeb5f4d53b..a8d85c9f37 100644 --- a/include/link.h +++ b/include/link.h @@ -60,6 +60,7 @@ struct r_search_path_elem; /* Forward declaration. */ struct link_map; +struct dl_ifunc_relocation; /* Structure to describe a single list of scope elements. The lookup functions get passed an array of pointers to such structures. */ @@ -318,6 +319,11 @@ struct link_map ElfW(Addr) l_relro_addr; size_t l_relro_size; +#ifdef HAVE_IFUNC + /* Deferred IFUNC relocations. Only used during relocation. */ + struct dl_ifunc_relocation *l_ifunc_relocations; +#endif + unsigned long long int l_serial; /* Audit information. This array apparent must be the last in the diff --git a/sysdeps/x86_64/dl-ifunc-reloc.h b/sysdeps/x86_64/dl-ifunc-reloc.h new file mode 100644 index 0000000000..bf436bf71c --- /dev/null +++ b/sysdeps/x86_64/dl-ifunc-reloc.h @@ -0,0 +1,91 @@ +/* IFUNC relocation processing, x86-64 version. + Copyright (C) 2001-2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Andreas Jaeger <aj@suse.de>. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef DL_IFUNC_RELOC_H +#define DL_IFUNC_RELOC_H + +static inline void +_dl_ifunc_process_relocation (const struct dl_ifunc_relocation *ifunc, + struct link_map *sym_map) +{ + const ElfW(Rela) *reloc = ifunc->reloc; + const unsigned long int r_type = ELFW(R_TYPE) (reloc->r_info); + ElfW(Addr) *const reloc_addr = ifunc->reloc_addr; + + /* Special case: A relative IFUNC relocation does not have an + associated symbol. */ + if (r_type == R_X86_64_IRELATIVE) + { + ElfW(Addr) value = sym_map->l_addr + reloc->r_addend; + value = ((ElfW(Addr) (*) (void)) value) (); + *reloc_addr = value; + return; + } + + ElfW(Addr) value = (ElfW(Addr)) sym_map->l_addr + ifunc->ifunc_sym->st_value; + value = ((ElfW(Addr) (*) (void)) value) (); + + /* This switch statement needs to be kept in sync with the switch + statement in elf_machine_rela. */ + switch (r_type) + { + case R_X86_64_GLOB_DAT: + case R_X86_64_JUMP_SLOT: + *reloc_addr = value + reloc->r_addend; + break; + + case R_X86_64_64: + /* value + r_addend may be > 0xffffffff and R_X86_64_64 + relocation updates the whole 64-bit entry. */ + *(Elf64_Addr *) reloc_addr = (Elf64_Addr) value + reloc->r_addend; + break; + case R_X86_64_32: + value += reloc->r_addend; + *(unsigned int *) reloc_addr = value; + + const char *fmt; + if (__glibc_unlikely (value > UINT_MAX)) + { + const char *strtab; + + fmt = "\ +%s: Symbol `%s' causes overflow in R_X86_64_32 relocation\n"; + print_err: + strtab = (const char *) D_PTR (sym_map, l_info[DT_STRTAB]); + + _dl_error_printf (fmt, RTLD_PROGNAME, + strtab + ifunc->ifunc_sym->st_name); + } + break; + case R_X86_64_PC32: + value += reloc->r_addend - (ElfW(Addr)) reloc_addr; + *(unsigned int *) reloc_addr = value; + if (__glibc_unlikely (value != (int) value)) + { + fmt = "\ +%s: Symbol `%s' causes overflow in R_X86_64_PC32 relocation\n"; + goto print_err; + } + break; + default: + _dl_reloc_bad_type (ifunc->reloc_map, r_type, 0); + } +} + +#endif /* DL_IFUNC_RELOC_H */ diff --git a/sysdeps/x86_64/dl-machine.h b/sysdeps/x86_64/dl-machine.h index 3e7ae22c67..02b6703b2e 100644 --- a/sysdeps/x86_64/dl-machine.h +++ b/sysdeps/x86_64/dl-machine.h @@ -27,6 +27,7 @@ #include <tls.h> #include <dl-tlsdesc.h> #include <cpu-features.c> +#include <dl-ifunc.h> /* Return nonzero iff ELF header is compatible with the running host. */ static inline int __attribute__ ((unused)) @@ -326,29 +327,20 @@ elf_machine_rela (struct link_map *map, const ElfW(Rela) *reloc, ElfW(Addr) value = (sym == NULL ? 0 : (ElfW(Addr)) sym_map->l_addr + sym->st_value); +# ifndef RTLD_BOOTSTRAP if (sym != NULL && __builtin_expect (ELFW(ST_TYPE) (sym->st_info) == STT_GNU_IFUNC, 0) && __builtin_expect (sym->st_shndx != SHN_UNDEF, 1) && __builtin_expect (!skip_ifunc, 1)) { -# ifndef RTLD_BOOTSTRAP - if (sym_map != map - && sym_map->l_type != lt_executable - && !sym_map->l_relocated) - { - const char *strtab - = (const char *) D_PTR (map, l_info[DT_STRTAB]); - _dl_fatal_printf ("\ -%s: Relink `%s' with `%s' for IFUNC symbol `%s'\n", - RTLD_PROGNAME, map->l_name, - sym_map->l_name, - strtab + refsym->st_name); - } -# endif - value = ((ElfW(Addr) (*) (void)) value) (); + _dl_ifunc_record_reloc (map, reloc, reloc_addr, sym_map, sym); + return; } +# endif /* !RTLD_BOOTSTRAP */ + /* This switch statement needs to be kept in sync with the + switch statement in _dl_ifunc_process_relocation. */ switch (r_type) { # ifndef RTLD_BOOTSTRAP @@ -528,9 +520,7 @@ elf_machine_rela (struct link_map *map, const ElfW(Rela) *reloc, break; # endif case R_X86_64_IRELATIVE: - value = map->l_addr + reloc->r_addend; - value = ((ElfW(Addr) (*) (void)) value) (); - *reloc_addr = value; + _dl_ifunc_record_reloc (map, reloc, reloc_addr, map, NULL); break; default: _dl_reloc_bad_type (map, r_type, 0); |