diff options
author | Ben Gamari <ben@smart-cactus.org> | 2022-02-07 16:21:50 -0500 |
---|---|---|
committer | Marge Bot <ben+marge-bot@smart-cactus.org> | 2022-02-09 20:43:39 -0500 |
commit | e219ac826b05db833531028e0663f62f12eff010 (patch) | |
tree | c7b133cc689d1068066d356de12a6d1ae57c5e16 | |
parent | 3df06922f03191310ebee0547de1782eeb6bda67 (diff) | |
download | haskell-e219ac826b05db833531028e0663f62f12eff010.tar.gz |
rts: Move mmapForLinker and friends to linker/MMap.c
They are not particularly related to linking.
-rw-r--r-- | rts/ExecPage.c | 2 | ||||
-rw-r--r-- | rts/Linker.c | 252 | ||||
-rw-r--r-- | rts/LinkerInternals.h | 88 | ||||
-rw-r--r-- | rts/linker/Elf.c | 1 | ||||
-rw-r--r-- | rts/linker/LoadArchive.c | 1 | ||||
-rw-r--r-- | rts/linker/M32Alloc.c | 2 | ||||
-rw-r--r-- | rts/linker/MMap.c | 290 | ||||
-rw-r--r-- | rts/linker/MMap.h | 79 | ||||
-rw-r--r-- | rts/linker/SymbolExtras.c | 1 | ||||
-rw-r--r-- | rts/linker/elf_got.c | 1 | ||||
-rw-r--r-- | rts/rts.cabal.in | 1 |
11 files changed, 377 insertions, 341 deletions
diff --git a/rts/ExecPage.c b/rts/ExecPage.c index 24d4d65bad..0f83c8e1f5 100644 --- a/rts/ExecPage.c +++ b/rts/ExecPage.c @@ -6,8 +6,8 @@ */ #include "Rts.h" -#include "LinkerInternals.h" #include "sm/OSMem.h" +#include "linker/MMap.h" ExecPage *allocateExecPage() { ExecPage *page = (ExecPage *) mmapAnonForLinker(getPageSize()); diff --git a/rts/Linker.c b/rts/Linker.c index 8ac529465b..fcba191249 100644 --- a/rts/Linker.c +++ b/rts/Linker.c @@ -31,6 +31,7 @@ #include "linker/M32Alloc.h" #include "linker/CacheFlush.h" #include "linker/SymbolExtras.h" +#include "linker/MMap.h" #include "PathUtils.h" #include "CheckUnload.h" // createOCSectionIndices #include "ReportMemoryMap.h" @@ -172,8 +173,6 @@ Mutex linker_mutex; /* Generic wrapper function to try and Resolve and RunInit oc files */ int ocTryLoad( ObjectCode* oc ); -static void *mmap_32bit_base = LINKER_LOAD_BASE; - static void ghciRemoveSymbolTable(StrHashTable *table, const SymbolName* key, ObjectCode *owner) { @@ -1009,255 +1008,6 @@ resolveSymbolAddr (pathchar* buffer, int size, #endif /* OBJFORMAT_PEi386 */ } -static const char *memoryAccessDescription(MemoryAccess mode) -{ - switch (mode) { - case MEM_NO_ACCESS: return "no-access"; - case MEM_READ_ONLY: return "read-only"; - case MEM_READ_WRITE: return "read-write"; - case MEM_READ_EXECUTE: return "read-execute"; - default: barf("invalid MemoryAccess"); - } -} - -#if defined(mingw32_HOST_OS) - -// -// Returns NULL on failure. -// -void * -mmapAnonForLinker (size_t bytes) -{ - return VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); -} - -void -munmapForLinker (void *addr, size_t bytes, const char *caller) -{ - if (VirtualFree(addr, 0, MEM_RELEASE) == 0) { - sysErrorBelch("munmapForLinker: %s: Failed to unmap %zd bytes at %p", - caller, bytes, addr); - } -} - -/** - * Change the allowed access modes of a region of memory previously allocated - * with mmapAnonForLinker. - */ -void -mprotectForLinker(void *start, size_t len, MemoryAccess mode) -{ - DWORD old; - if (len == 0) { - return; - } - DWORD prot; - switch (mode) { - case MEM_NO_ACCESS: prot = PAGE_NOACCESS; break; - case MEM_READ_ONLY: prot = PAGE_READONLY; break; - case MEM_READ_WRITE: prot = PAGE_READWRITE; break; - case MEM_READ_EXECUTE: prot = PAGE_EXECUTE_READ; break; - default: barf("invalid MemoryAccess"); - } - - if (VirtualProtect(start, len, prot, &old) == 0) { - sysErrorBelch("mprotectForLinker: failed to protect %zd bytes at %p as %s", - len, start, memoryAccessDescription(mode)); - ASSERT(false); - } -} - -#elif RTS_LINKER_USE_MMAP -// -// Returns NULL on failure. -// -void * -mmapForLinker (size_t bytes, uint32_t prot, uint32_t flags, int fd, int offset) -{ - void *map_addr = NULL; - void *result; - size_t size; - uint32_t tryMap32Bit = RtsFlags.MiscFlags.linkerAlwaysPic - ? 0 - : TRY_MAP_32BIT; - static uint32_t fixed = 0; - - IF_DEBUG(linker_verbose, debugBelch("mmapForLinker: start\n")); - size = roundUpToPage(bytes); - -#if defined(MAP_LOW_MEM) -mmap_again: -#endif - - if (mmap_32bit_base != NULL) { - map_addr = mmap_32bit_base; - } - - IF_DEBUG(linker_verbose, - debugBelch("mmapForLinker: \tprotection %#0x\n", prot)); - IF_DEBUG(linker_verbose, - debugBelch("mmapForLinker: \tflags %#0x\n", - MAP_PRIVATE | tryMap32Bit | fixed | flags)); - IF_DEBUG(linker_verbose, - debugBelch("mmapForLinker: \tsize %#0zx\n", bytes)); - IF_DEBUG(linker_verbose, - debugBelch("mmapForLinker: \tmap_addr %p\n", map_addr)); - - result = mmap(map_addr, size, prot, - MAP_PRIVATE|tryMap32Bit|fixed|flags, fd, offset); - - if (result == MAP_FAILED) { - reportMemoryMap(); - sysErrorBelch("mmap %" FMT_Word " bytes at %p",(W_)size,map_addr); - errorBelch("Try specifying an address with +RTS -xm<addr> -RTS"); - return NULL; - } - -#if defined(MAP_LOW_MEM) - if (RtsFlags.MiscFlags.linkerAlwaysPic) { - /* make no attempt at mapping low memory if we are assuming PIC */ - } else if (mmap_32bit_base != NULL) { - if (result != map_addr) { - if ((W_)result > 0x80000000) { - // oops, we were given memory over 2Gb - munmap(result,size); -#if defined(freebsd_HOST_OS) || \ - defined(kfreebsdgnu_HOST_OS) || \ - defined(dragonfly_HOST_OS) - // Some platforms require MAP_FIXED. This is normally - // a bad idea, because MAP_FIXED will overwrite - // existing mappings. - fixed = MAP_FIXED; - goto mmap_again; -#else - reportMemoryMap(); - errorBelch("mmapForLinker: failed to mmap() memory below 2Gb; " - "asked for %lu bytes at %p. " - "Try specifying an address with +RTS -xm<addr> -RTS", - size, map_addr); - return NULL; -#endif - } else { - // hmm, we were given memory somewhere else, but it's - // still under 2Gb so we can use it. - } - } - } else { - if ((W_)result > 0x80000000) { - // oops, we were given memory over 2Gb - // ... try allocating memory somewhere else?; - debugTrace(DEBUG_linker, - "MAP_32BIT didn't work; gave us %lu bytes at 0x%p", - bytes, result); - munmap(result, size); - - // Set a base address and try again... (guess: 1Gb) - mmap_32bit_base = (void*)0x40000000; - goto mmap_again; - } - } -#elif (defined(aarch64_TARGET_ARCH) || defined(aarch64_HOST_ARCH)) - // for aarch64 we need to make sure we stay within 4GB of the - // mmap_32bit_base, and we also do not want to update it. - if (result != map_addr) { - // upper limit 4GB - size of the object file - 1mb wiggle room. - if(llabs((uintptr_t)result - (uintptr_t)&stg_upd_frame_info) > (2<<32) - size - (2<<20)) { - // not within range :( - debugTrace(DEBUG_linker, - "MAP_32BIT didn't work; gave us %lu bytes at 0x%p", - bytes, result); - munmap(result, size); - // TODO: some abort/mmap_32bit_base recomputation based on - // if mmap_32bit_base is changed, or still at stg_upd_frame_info - goto mmap_again; - } - } -#endif - - if (mmap_32bit_base != NULL) { - // Next time, ask for memory right after our new mapping to maximize the - // chance that we get low memory. - mmap_32bit_base = (void*) ((uintptr_t)result + size); - } - - IF_DEBUG(linker_verbose, - debugBelch("mmapForLinker: mapped %" FMT_Word - " bytes starting at %p\n", (W_)size, result)); - IF_DEBUG(linker_verbose, - debugBelch("mmapForLinker: done\n")); - - return result; -} - -/* - * Map read/write pages in low memory. Returns NULL on failure. - */ -void * -mmapAnonForLinker (size_t bytes) -{ - return mmapForLinker (bytes, PROT_READ|PROT_WRITE, MAP_ANONYMOUS, -1, 0); -} - -void munmapForLinker (void *addr, size_t bytes, const char *caller) -{ - int r = munmap(addr, bytes); - if (r == -1) { - // Should we abort here? - sysErrorBelch("munmap: %s", caller); - } -} - -/* Note [Memory protection in the linker] - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * For many years the linker would simply map all of its memory - * with PROT_READ|PROT_WRITE|PROT_EXEC. However operating systems have been - * becoming increasingly reluctant to accept this practice (e.g. #17353, - * #12657) and for good reason: writable code is ripe for exploitation. - * - * Consequently mmapForLinker now maps its memory with PROT_READ|PROT_WRITE. - * After the linker has finished filling/relocating the mapping it must then - * call mprotectForLinker on the sections of the mapping which - * contain executable code. - * - * Note that the m32 allocator handles protection of its allocations. For this - * reason the caller to m32_alloc() must tell the allocator whether the - * allocation needs to be executable. The caller must then ensure that they - * call m32_allocator_flush() after they are finished filling the region, which - * will cause the allocator to change the protection bits to - * PROT_READ|PROT_EXEC. - * - */ - -/* - * Mark an portion of a mapping previously reserved by mmapForLinker - * as executable (but not writable). - */ -void mprotectForLinker(void *start, size_t len, MemoryAccess mode) -{ - if (len == 0) { - return; - } - IF_DEBUG(linker_verbose, - debugBelch("mprotectForLinker: protecting %" FMT_Word - " bytes starting at %p as %s\n", - (W_)len, start, memoryAccessDescription(mode))); - - int prot; - switch (mode) { - case MEM_NO_ACCESS: prot = 0; break; - case MEM_READ_ONLY: prot = PROT_READ; break; - case MEM_READ_WRITE: prot = PROT_READ | PROT_WRITE; break; - case MEM_READ_EXECUTE: prot = PROT_READ | PROT_EXEC; break; - default: barf("invalid MemoryAccess"); - } - - if (mprotect(start, len, prot) == -1) { - sysErrorBelch("mprotectForLinker: failed to protect %zd bytes at %p as %s", - len, start, memoryAccessDescription(mode)); - } -} -#endif - /* * Remove symbols from the symbol table, and free oc->symbols. * This operation is idempotent. diff --git a/rts/LinkerInternals.h b/rts/LinkerInternals.h index af72ad587a..2582b851cc 100644 --- a/rts/LinkerInternals.h +++ b/rts/LinkerInternals.h @@ -375,19 +375,6 @@ void exitLinker( void ); void freeObjectCode (ObjectCode *oc); SymbolAddr* loadSymbol(SymbolName *lbl, RtsSymbolInfo *pinfo); -/** Access modes for mprotectForLinker */ -typedef enum { - MEM_NO_ACCESS, - MEM_READ_ONLY, - MEM_READ_WRITE, - MEM_READ_EXECUTE, -} MemoryAccess; - -void *mmapAnonForLinker (size_t bytes); -void *mmapForLinker (size_t bytes, uint32_t prot, uint32_t flags, int fd, int offset); -void mprotectForLinker(void *start, size_t len, MemoryAccess mode); -void munmapForLinker (void *addr, size_t bytes, const char *caller); - void addProddableBlock ( ObjectCode* oc, void* start, int size ); void checkProddableBlock (ObjectCode *oc, void *addr, size_t size ); void freeProddableBlocks (ObjectCode *oc); @@ -442,65 +429,6 @@ resolveSymbolAddr (pathchar* buffer, int size, #define USE_CONTIGUOUS_MMAP 0 #endif -/* Link objects into the lower 2Gb on x86_64 and AArch64. GHC assumes the - * small memory model on this architecture (see gcc docs, - * -mcmodel=small). - * - * MAP_32BIT not available on OpenBSD/amd64 - */ -#if defined(MAP_32BIT) && (defined(x86_64_HOST_ARCH) || (defined(aarch64_TARGET_ARCH) || defined(aarch64_HOST_ARCH))) -#define MAP_LOW_MEM -#define TRY_MAP_32BIT MAP_32BIT -#else -#define TRY_MAP_32BIT 0 -#endif - -#if defined(aarch64_HOST_ARCH) -// On AArch64 MAP_32BIT is not available but we are still bound by the small -// memory model. Consequently we still try using the MAP_LOW_MEM allocation -// strategy. -#define MAP_LOW_MEM -#endif - -/* - * Note [MAP_LOW_MEM] - * ~~~~~~~~~~~~~~~~~~ - * Due to the small memory model (see above), on x86_64 and AArch64 we have to - * map all our non-PIC object files into the low 2Gb of the address space (why - * 2Gb and not 4Gb? Because all addresses must be reachable using a 32-bit - * signed PC-relative offset). On x86_64 Linux we can do this using the - * MAP_32BIT flag to mmap(), however on other OSs (e.g. *BSD, see #2063, and - * also on Linux inside Xen, see #2512), we can't do this. So on these - * systems, we have to pick a base address in the low 2Gb of the address space - * and try to allocate memory from there. - * - * The same holds for aarch64, where the default, even with PIC, model - * is 4GB. The linker is free to emit AARCH64_ADR_PREL_PG_HI21 - * relocations. - * - * We pick a default address based on the OS, but also make this - * configurable via an RTS flag (+RTS -xm) - */ - -#if defined(aarch64_TARGET_ARCH) || defined(aarch64_HOST_ARCH) -// Try to use stg_upd_frame_info as the base. We need to be within +-4GB of that -// address, otherwise we violate the aarch64 memory model. Any object we load -// can potentially reference any of the ones we bake into the binary (and list) -// in RtsSymbols. Thus we'll need to be within +-4GB of those, -// stg_upd_frame_info is a good candidate as it's referenced often. -#define LINKER_LOAD_BASE ((void *) &stg_upd_frame_info) -#elif defined(x86_64_HOST_ARCH) && defined(mingw32_HOST_OS) -// On Windows (which now uses high-entropy ASLR by default) we need to ensure -// that we map code near the executable image. We use stg_upd_frame_info as a -// proxy for the image location. -#define LINKER_LOAD_BASE ((void *) &stg_upd_frame_info) -#elif defined(MAP_32BIT) || DEFAULT_LINKER_ALWAYS_PIC -// Try to use MAP_32BIT -#define LINKER_LOAD_BASE ((void *) 0x0) -#else -// A guess: 1 GB. -#define LINKER_LOAD_BASE ((void *) 0x40000000) -#endif HsInt isAlreadyLoaded( pathchar *path ); OStatus getObjectLoadStatus_ (pathchar *path); @@ -513,20 +441,4 @@ ObjectCode* mkOc( ObjectType type, pathchar *path, char *image, int imageSize, void initSegment(Segment *s, void *start, size_t size, SegmentProt prot, int n_sections); void freeSegments(ObjectCode *oc); -/* MAP_ANONYMOUS is MAP_ANON on some systems, - e.g. OS X (before Sierra), OpenBSD etc */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -/* In order to simplify control flow a bit, some references to mmap-related - definitions are blocked off by a C-level if statement rather than a CPP-level - #if statement. Since those are dead branches when !RTS_LINKER_USE_MMAP, we - just stub out the relevant symbols here -*/ -#if !RTS_LINKER_USE_MMAP -#define munmap(x,y) /* nothing */ -#define MAP_ANONYMOUS 0 -#endif - #include "EndPrivate.h" diff --git a/rts/linker/Elf.c b/rts/linker/Elf.c index 10ac5d2896..f84d6bb495 100644 --- a/rts/linker/Elf.c +++ b/rts/linker/Elf.c @@ -17,6 +17,7 @@ #include "RtsSymbolInfo.h" #include "CheckUnload.h" #include "LinkerInternals.h" +#include "linker/MMap.h" #include "linker/Elf.h" #include "linker/CacheFlush.h" #include "linker/M32Alloc.h" diff --git a/rts/linker/LoadArchive.c b/rts/linker/LoadArchive.c index ff8630d57e..99e405db22 100644 --- a/rts/linker/LoadArchive.c +++ b/rts/linker/LoadArchive.c @@ -7,6 +7,7 @@ #include "LinkerInternals.h" #include "CheckUnload.h" // loaded_objects, insertOCSectionIndices #include "linker/M32Alloc.h" +#include "linker/MMap.h" /* Platform specific headers */ #if defined(OBJFORMAT_PEi386) diff --git a/rts/linker/M32Alloc.c b/rts/linker/M32Alloc.c index 2c3b1b48fb..5ce0342482 100644 --- a/rts/linker/M32Alloc.c +++ b/rts/linker/M32Alloc.c @@ -10,7 +10,7 @@ #include "sm/OSMem.h" #include "RtsUtils.h" #include "linker/M32Alloc.h" -#include "LinkerInternals.h" +#include "linker/MMap.h" #include "ReportMemoryMap.h" #include <inttypes.h> diff --git a/rts/linker/MMap.c b/rts/linker/MMap.c new file mode 100644 index 0000000000..f4eaf3d686 --- /dev/null +++ b/rts/linker/MMap.c @@ -0,0 +1,290 @@ +#include "Rts.h" + +#include "sm/OSMem.h" +#include "linker/MMap.h" +#include "Trace.h" +#include "ReportMemoryMap.h" + +#if RTS_LINKER_USE_MMAP +#include <sys/mman.h> +#endif + +/* Link objects into the lower 2Gb on x86_64 and AArch64. GHC assumes the + * small memory model on this architecture (see gcc docs, + * -mcmodel=small). + * + * MAP_32BIT not available on OpenBSD/amd64 + */ +#if defined(MAP_32BIT) && (defined(x86_64_HOST_ARCH) || (defined(aarch64_TARGET_ARCH) || defined(aarch64_HOST_ARCH))) +#define MAP_LOW_MEM +#define TRY_MAP_32BIT MAP_32BIT +#else +#define TRY_MAP_32BIT 0 +#endif + +/* MAP_ANONYMOUS is MAP_ANON on some systems, + e.g. OS X (before Sierra), OpenBSD etc */ +#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) +#define MAP_ANONYMOUS MAP_ANON +#endif + +/* In order to simplify control flow a bit, some references to mmap-related + definitions are blocked off by a C-level if statement rather than a CPP-level + #if statement. Since those are dead branches when !RTS_LINKER_USE_MMAP, we + just stub out the relevant symbols here +*/ +#if !RTS_LINKER_USE_MMAP +#define munmap(x,y) /* nothing */ +#define MAP_ANONYMOUS 0 +#endif + +void *mmap_32bit_base = LINKER_LOAD_BASE; + +static const char *memoryAccessDescription(MemoryAccess mode) +{ + switch (mode) { + case MEM_NO_ACCESS: return "no-access"; + case MEM_READ_ONLY: return "read-only"; + case MEM_READ_WRITE: return "read-write"; + case MEM_READ_EXECUTE: return "read-execute"; + default: barf("invalid MemoryAccess"); + } +} + +#if defined(mingw32_HOST_OS) + +// +// Returns NULL on failure. +// +void * +mmapAnonForLinker (size_t bytes) +{ + return VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); +} + +void +munmapForLinker (void *addr, size_t bytes, const char *caller) +{ + if (VirtualFree(addr, 0, MEM_RELEASE) == 0) { + sysErrorBelch("munmapForLinker: %s: Failed to unmap %zd bytes at %p", + caller, bytes, addr); + } +} + +/** + * Change the allowed access modes of a region of memory previously allocated + * with mmapAnonForLinker. + */ +void +mprotectForLinker(void *start, size_t len, MemoryAccess mode) +{ + DWORD old; + if (len == 0) { + return; + } + DWORD prot; + switch (mode) { + case MEM_NO_ACCESS: prot = PAGE_NOACCESS; break; + case MEM_READ_ONLY: prot = PAGE_READONLY; break; + case MEM_READ_WRITE: prot = PAGE_READWRITE; break; + case MEM_READ_EXECUTE: prot = PAGE_EXECUTE_READ; break; + default: barf("invalid MemoryAccess"); + } + + if (VirtualProtect(start, len, prot, &old) == 0) { + sysErrorBelch("mprotectForLinker: failed to protect %zd bytes at %p as %s", + len, start, memoryAccessDescription(mode)); + ASSERT(false); + } +} + +#elif RTS_LINKER_USE_MMAP +// +// Returns NULL on failure. +// +void * +mmapForLinker (size_t bytes, uint32_t prot, uint32_t flags, int fd, int offset) +{ + void *map_addr = NULL; + void *result; + size_t size; + uint32_t tryMap32Bit = RtsFlags.MiscFlags.linkerAlwaysPic + ? 0 + : TRY_MAP_32BIT; + static uint32_t fixed = 0; + + IF_DEBUG(linker_verbose, debugBelch("mmapForLinker: start\n")); + size = roundUpToPage(bytes); + +#if defined(MAP_LOW_MEM) +mmap_again: +#endif + + if (mmap_32bit_base != NULL) { + map_addr = mmap_32bit_base; + } + + IF_DEBUG(linker_verbose, + debugBelch("mmapForLinker: \tprotection %#0x\n", prot)); + IF_DEBUG(linker_verbose, + debugBelch("mmapForLinker: \tflags %#0x\n", + MAP_PRIVATE | tryMap32Bit | fixed | flags)); + IF_DEBUG(linker_verbose, + debugBelch("mmapForLinker: \tsize %#0zx\n", bytes)); + IF_DEBUG(linker_verbose, + debugBelch("mmapForLinker: \tmap_addr %p\n", map_addr)); + + result = mmap(map_addr, size, prot, + MAP_PRIVATE|tryMap32Bit|fixed|flags, fd, offset); + + if (result == MAP_FAILED) { + reportMemoryMap(); + sysErrorBelch("mmap %" FMT_Word " bytes at %p",(W_)size,map_addr); + errorBelch("Try specifying an address with +RTS -xm<addr> -RTS"); + return NULL; + } + +#if defined(MAP_LOW_MEM) + if (RtsFlags.MiscFlags.linkerAlwaysPic) { + /* make no attempt at mapping low memory if we are assuming PIC */ + } else if (mmap_32bit_base != NULL) { + if (result != map_addr) { + if ((W_)result > 0x80000000) { + // oops, we were given memory over 2Gb + munmap(result,size); +#if defined(freebsd_HOST_OS) || \ + defined(kfreebsdgnu_HOST_OS) || \ + defined(dragonfly_HOST_OS) + // Some platforms require MAP_FIXED. This is normally + // a bad idea, because MAP_FIXED will overwrite + // existing mappings. + fixed = MAP_FIXED; + goto mmap_again; +#else + reportMemoryMap(); + errorBelch("mmapForLinker: failed to mmap() memory below 2Gb; " + "asked for %lu bytes at %p. " + "Try specifying an address with +RTS -xm<addr> -RTS", + size, map_addr); + return NULL; +#endif + } else { + // hmm, we were given memory somewhere else, but it's + // still under 2Gb so we can use it. + } + } + } else { + if ((W_)result > 0x80000000) { + // oops, we were given memory over 2Gb + // ... try allocating memory somewhere else?; + debugTrace(DEBUG_linker, + "MAP_32BIT didn't work; gave us %lu bytes at 0x%p", + bytes, result); + munmap(result, size); + + // Set a base address and try again... (guess: 1Gb) + mmap_32bit_base = (void*)0x40000000; + goto mmap_again; + } + } +#elif (defined(aarch64_TARGET_ARCH) || defined(aarch64_HOST_ARCH)) + // for aarch64 we need to make sure we stay within 4GB of the + // mmap_32bit_base, and we also do not want to update it. + if (result != map_addr) { + // upper limit 4GB - size of the object file - 1mb wiggle room. + if(llabs((uintptr_t)result - (uintptr_t)&stg_upd_frame_info) > (2<<32) - size - (2<<20)) { + // not within range :( + debugTrace(DEBUG_linker, + "MAP_32BIT didn't work; gave us %lu bytes at 0x%p", + bytes, result); + munmap(result, size); + // TODO: some abort/mmap_32bit_base recomputation based on + // if mmap_32bit_base is changed, or still at stg_upd_frame_info + goto mmap_again; + } + } +#endif + + if (mmap_32bit_base != NULL) { + // Next time, ask for memory right after our new mapping to maximize the + // chance that we get low memory. + mmap_32bit_base = (void*) ((uintptr_t)result + size); + } + + IF_DEBUG(linker_verbose, + debugBelch("mmapForLinker: mapped %" FMT_Word + " bytes starting at %p\n", (W_)size, result)); + IF_DEBUG(linker_verbose, + debugBelch("mmapForLinker: done\n")); + + return result; +} + +/* + * Map read/write pages in low memory. Returns NULL on failure. + */ +void * +mmapAnonForLinker (size_t bytes) +{ + return mmapForLinker (bytes, PROT_READ|PROT_WRITE, MAP_ANONYMOUS, -1, 0); +} + +void munmapForLinker (void *addr, size_t bytes, const char *caller) +{ + int r = munmap(addr, bytes); + if (r == -1) { + // Should we abort here? + sysErrorBelch("munmap: %s", caller); + } +} + +/* Note [Memory protection in the linker] + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * For many years the linker would simply map all of its memory + * with PROT_READ|PROT_WRITE|PROT_EXEC. However operating systems have been + * becoming increasingly reluctant to accept this practice (e.g. #17353, + * #12657) and for good reason: writable code is ripe for exploitation. + * + * Consequently mmapForLinker now maps its memory with PROT_READ|PROT_WRITE. + * After the linker has finished filling/relocating the mapping it must then + * call mprotectForLinker on the sections of the mapping which + * contain executable code. + * + * Note that the m32 allocator handles protection of its allocations. For this + * reason the caller to m32_alloc() must tell the allocator whether the + * allocation needs to be executable. The caller must then ensure that they + * call m32_allocator_flush() after they are finished filling the region, which + * will cause the allocator to change the protection bits to + * PROT_READ|PROT_EXEC. + * + */ + +/* + * Mark an portion of a mapping previously reserved by mmapForLinker + * as executable (but not writable). + */ +void mprotectForLinker(void *start, size_t len, MemoryAccess mode) +{ + if (len == 0) { + return; + } + IF_DEBUG(linker_verbose, + debugBelch("mprotectForLinker: protecting %" FMT_Word + " bytes starting at %p as %s\n", + (W_)len, start, memoryAccessDescription(mode))); + + int prot; + switch (mode) { + case MEM_NO_ACCESS: prot = 0; break; + case MEM_READ_ONLY: prot = PROT_READ; break; + case MEM_READ_WRITE: prot = PROT_READ | PROT_WRITE; break; + case MEM_READ_EXECUTE: prot = PROT_READ | PROT_EXEC; break; + default: barf("invalid MemoryAccess"); + } + + if (mprotect(start, len, prot) == -1) { + sysErrorBelch("mprotectForLinker: failed to protect %zd bytes at %p as %s", + len, start, memoryAccessDescription(mode)); + } +} +#endif diff --git a/rts/linker/MMap.h b/rts/linker/MMap.h new file mode 100644 index 0000000000..ed0baa6899 --- /dev/null +++ b/rts/linker/MMap.h @@ -0,0 +1,79 @@ +#pragma once + +#include "BeginPrivate.h" + +#if defined(aarch64_HOST_ARCH) +// On AArch64 MAP_32BIT is not available but we are still bound by the small +// memory model. Consequently we still try using the MAP_LOW_MEM allocation +// strategy. +#define MAP_LOW_MEM +#endif + +/* + * Note [MAP_LOW_MEM] + * ~~~~~~~~~~~~~~~~~~ + * Due to the small memory model (see above), on x86_64 and AArch64 we have to + * map all our non-PIC object files into the low 2Gb of the address space (why + * 2Gb and not 4Gb? Because all addresses must be reachable using a 32-bit + * signed PC-relative offset). On x86_64 Linux we can do this using the + * MAP_32BIT flag to mmap(), however on other OSs (e.g. *BSD, see #2063, and + * also on Linux inside Xen, see #2512), we can't do this. So on these + * systems, we have to pick a base address in the low 2Gb of the address space + * and try to allocate memory from there. + * + * The same holds for aarch64, where the default, even with PIC, model + * is 4GB. The linker is free to emit AARCH64_ADR_PREL_PG_HI21 + * relocations. + * + * We pick a default address based on the OS, but also make this + * configurable via an RTS flag (+RTS -xm) + */ + +#if defined(aarch64_TARGET_ARCH) || defined(aarch64_HOST_ARCH) +// Try to use stg_upd_frame_info as the base. We need to be within +-4GB of that +// address, otherwise we violate the aarch64 memory model. Any object we load +// can potentially reference any of the ones we bake into the binary (and list) +// in RtsSymbols. Thus we'll need to be within +-4GB of those, +// stg_upd_frame_info is a good candidate as it's referenced often. +#define LINKER_LOAD_BASE ((void *) &stg_upd_frame_info) +#elif defined(x86_64_HOST_ARCH) && defined(mingw32_HOST_OS) +// On Windows (which now uses high-entropy ASLR by default) we need to ensure +// that we map code near the executable image. We use stg_upd_frame_info as a +// proxy for the image location. +#define LINKER_LOAD_BASE ((void *) &stg_upd_frame_info) +#elif defined(MAP_32BIT) || DEFAULT_LINKER_ALWAYS_PIC +// Try to use MAP_32BIT +#define LINKER_LOAD_BASE ((void *) 0x0) +#else +// A guess: 1 GB. +#define LINKER_LOAD_BASE ((void *) 0x40000000) +#endif + +/** Access modes for mprotectForLinker */ +typedef enum { + MEM_NO_ACCESS, + MEM_READ_ONLY, + MEM_READ_WRITE, + MEM_READ_EXECUTE, +} MemoryAccess; + +extern void *mmap_32bit_base; + +// Map read/write anonymous memory. +void *mmapAnonForLinker (size_t bytes); + +// Change protection of previous mapping memory. +void mprotectForLinker(void *start, size_t len, MemoryAccess mode); + +// Release a mapping. +void munmapForLinker (void *addr, size_t bytes, const char *caller); + +#if !defined(mingw32_HOST_OS) +// Map a file. +// +// Note that this not available on Windows since file mapping on Windows is +// sufficiently different to warrant its own interface. +void *mmapForLinker (size_t bytes, uint32_t prot, uint32_t flags, int fd, int offset); +#endif + +#include "EndPrivate.h" diff --git a/rts/linker/SymbolExtras.c b/rts/linker/SymbolExtras.c index 5c04e9b3a8..88192d43d9 100644 --- a/rts/linker/SymbolExtras.c +++ b/rts/linker/SymbolExtras.c @@ -10,6 +10,7 @@ */ #include "LinkerInternals.h" +#include "linker/MMap.h" #if defined(NEED_SYMBOL_EXTRAS) #if !defined(x86_64_HOST_ARCH) || !defined(mingw32_HOST_OS) diff --git a/rts/linker/elf_got.c b/rts/linker/elf_got.c index ae75329295..eefdae34c6 100644 --- a/rts/linker/elf_got.c +++ b/rts/linker/elf_got.c @@ -1,5 +1,6 @@ #include "Rts.h" #include "elf_got.h" +#include "linker/MMap.h" #include <string.h> diff --git a/rts/rts.cabal.in b/rts/rts.cabal.in index b9412327ed..e2abeb5df1 100644 --- a/rts/rts.cabal.in +++ b/rts/rts.cabal.in @@ -547,6 +547,7 @@ library linker/Elf.c linker/LoadArchive.c linker/M32Alloc.c + linker/MMap.c linker/MachO.c linker/macho/plt.c linker/macho/plt_aarch64.c |