diff options
Diffstat (limited to 'js/src/nanojit/CodeAlloc.cpp')
| -rw-r--r-- | js/src/nanojit/CodeAlloc.cpp | 570 |
1 files changed, 570 insertions, 0 deletions
diff --git a/js/src/nanojit/CodeAlloc.cpp b/js/src/nanojit/CodeAlloc.cpp new file mode 100644 index 0000000..61b711a --- /dev/null +++ b/js/src/nanojit/CodeAlloc.cpp @@ -0,0 +1,570 @@ +/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */ +/* vi: set ts=4 sw=4 expandtab: (add to ~/.vimrc: set modeline modelines=5) */ +/* ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is [Open Source Virtual Machine]. + * + * The Initial Developer of the Original Code is + * Adobe System Incorporated. + * Portions created by the Initial Developer are Copyright (C) 2004-2007 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Adobe AS3 Team + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#include "nanojit.h" + +//#define DOPROF +#include "../vprof/vprof.h" + +#ifdef FEATURE_NANOJIT + +namespace nanojit +{ + static const bool verbose = false; +#ifdef VMCFG_VTUNE + // vtune jit profiling api can't handle non-contiguous methods, + // so make the allocation size huge to avoid non-contiguous methods + static const int pagesPerAlloc = 128; // 1MB +#elif defined(NANOJIT_ARM) + // ARM requires single-page allocations, due to the constant pool that + // lives on each page that must be reachable by a 4kb pcrel load. + static const int pagesPerAlloc = 1; +#else + static const int pagesPerAlloc = 16; +#endif + + CodeAlloc::CodeAlloc() + : heapblocks(0) + , availblocks(0) + , totalAllocated(0) + , bytesPerPage(VMPI_getVMPageSize()) + , bytesPerAlloc(pagesPerAlloc * bytesPerPage) + { + } + + CodeAlloc::~CodeAlloc() { + reset(); + } + + void CodeAlloc::reset() { + // give all memory back to gcheap. Assumption is that all + // code is done being used by now. + for (CodeList* hb = heapblocks; hb != 0; ) { + _nvprof("free page",1); + CodeList* next = hb->next; + CodeList* fb = firstBlock(hb); + markBlockWrite(fb); + freeCodeChunk(fb, bytesPerAlloc); + totalAllocated -= bytesPerAlloc; + hb = next; + } + NanoAssert(!totalAllocated); + heapblocks = availblocks = 0; + } + + CodeList* CodeAlloc::firstBlock(CodeList* term) { + // use uintptr_t, rather than char*, to avoid "increases required alignment" warning + uintptr_t end = (uintptr_t)alignUp(term, bytesPerPage); + return (CodeList*) (end - (uintptr_t)bytesPerAlloc); + } + + static int round(size_t x) { + return (int)((x + 512) >> 10); + } + + void CodeAlloc::logStats() { + size_t total = 0; + size_t frag_size = 0; + size_t free_size = 0; + int free_count = 0; + for (CodeList* hb = heapblocks; hb != 0; hb = hb->next) { + total += bytesPerAlloc; + for (CodeList* b = hb->lower; b != 0; b = b->lower) { + if (b->isFree) { + free_count++; + free_size += b->blockSize(); + if (b->size() < minAllocSize) + frag_size += b->blockSize(); + } + } + } + avmplus::AvmLog("code-heap: %dk free %dk fragmented %d\n", + round(total), round(free_size), frag_size); + } + + inline void CodeAlloc::markBlockWrite(CodeList* b) { + NanoAssert(b->terminator != NULL); + CodeList* term = b->terminator; + if (term->isExec) { + markCodeChunkWrite(firstBlock(term), bytesPerAlloc); + term->isExec = false; + } + } + + void CodeAlloc::alloc(NIns* &start, NIns* &end, size_t byteLimit) { + if (!availblocks) { + // no free mem, get more + addMem(); + } + + // grab a block + NanoAssert(!byteLimit || byteLimit > blkSpaceFor(2)); // if a limit is imposed it must be bigger than 2x minimum block size (see below) + markBlockWrite(availblocks); + CodeList* b = removeBlock(availblocks); + + // limit imposed (byteLimit > 0) and the block is too big? then break it apart + if (byteLimit > 0 && b->size() > byteLimit) { + + size_t consume; // # bytes to extract from the free block + + // enough space to carve out a perfectly sized blk? (leaving at least a full free blk) + if (b->size() >= byteLimit + headerSpaceFor(1) + blkSpaceFor(1)) { + // yes, then take exactly what we need + consume = byteLimit + headerSpaceFor(1); + } else { + // no, then we should only take the min amount + consume = blkSpaceFor(1); + + // ... and since b->size() > byteLimit && byteLimit > blkSpaceFor(2) + NanoAssert( b->size() > blkSpaceFor(2) ); + NanoAssert( b->size() - consume > blkSpaceFor(1) ); // thus, we know that at least 1 blk left. + } + + // break block into 2 pieces, returning the lower portion to the free list + CodeList* higher = b->higher; + b->end = (NIns*) ( (uintptr_t)b->end - consume ); + CodeList* b1 = b->higher; + higher->lower = b1; + b1->higher = higher; + b1->lower = b; + b1->terminator = b->terminator; + NanoAssert(b->size() > minAllocSize); + addBlock(availblocks, b); // put back the rest of the block + b = b1; + } + NanoAssert(b->size() >= minAllocSize); + b->next = 0; // not technically needed (except for debug builds), but good hygiene. + b->isFree = false; + start = b->start(); + end = b->end; + if (verbose) + avmplus::AvmLog("CodeAlloc(%p).alloc %p-%p %d\n", this, start, end, int(end-start)); + debug_only(sanity_check();) + } + + void CodeAlloc::free(NIns* start, NIns *end) { + NanoAssert(heapblocks); + CodeList *blk = getBlock(start, end); + if (verbose) + avmplus::AvmLog("free %p-%p %d\n", start, end, (int)blk->size()); + + NanoAssert(!blk->isFree); + + // coalesce adjacent blocks. + bool already_on_avail_list; + + if (blk->lower && blk->lower->isFree) { + // combine blk into blk->lower (destroy blk) + CodeList* lower = blk->lower; + CodeList* higher = blk->higher; + already_on_avail_list = lower->size() >= minAllocSize; + lower->higher = higher; + higher->lower = lower; + blk = lower; + } + else + already_on_avail_list = false; + + // the last block in each heapblock is a terminator block, + // which is never free, therefore blk->higher != null + if (blk->higher->isFree) { + CodeList *higher = blk->higher->higher; + CodeList *coalescedBlock = blk->higher; + + if ( coalescedBlock->size() >= minAllocSize ) { + // Unlink coalescedBlock from the available block chain. + if ( availblocks == coalescedBlock ) { + removeBlock(availblocks); + } + else { + CodeList* free_block = availblocks; + while ( free_block && free_block->next != coalescedBlock) { + NanoAssert(free_block->size() >= minAllocSize); + NanoAssert(free_block->isFree); + NanoAssert(free_block->next); + free_block = free_block->next; + } + NanoAssert(free_block && free_block->next == coalescedBlock); + free_block->next = coalescedBlock->next; + } + } + + // combine blk->higher into blk (destroy coalescedBlock) + blk->higher = higher; + higher->lower = blk; + } + blk->isFree = true; + NanoAssert(!blk->lower || !blk->lower->isFree); + NanoAssert(blk->higher && !blk->higher->isFree); + //memset(blk->start(), 0xCC, blk->size()); // INT 3 instruction + if ( !already_on_avail_list && blk->size() >= minAllocSize ) + addBlock(availblocks, blk); + + NanoAssert(heapblocks); + debug_only(sanity_check();) + } + + void CodeAlloc::freeAll(CodeList* &code) { + while (code) { + CodeList *b = removeBlock(code); + free(b->start(), b->end); + } + } + +#if defined NANOJIT_ARM && defined UNDER_CE + // Use a single flush for the whole CodeList, when we have no + // finer-granularity flush support, as on WinCE. + void CodeAlloc::flushICache(CodeList* &/*blocks*/) { + FlushInstructionCache(GetCurrentProcess(), NULL, NULL); + } +#else + void CodeAlloc::flushICache(CodeList* &blocks) { + for (CodeList *b = blocks; b != 0; b = b->next) + flushICache(b->start(), b->size()); + } +#endif + +#if defined(AVMPLUS_UNIX) && defined(NANOJIT_ARM) +#include <asm/unistd.h> +extern "C" void __clear_cache(char *BEG, char *END); +#endif + +#if defined(AVMPLUS_UNIX) && defined(NANOJIT_MIPS) +#include <asm/cachectl.h> +extern "C" int cacheflush(char *addr, int nbytes, int cache); +#endif + +#ifdef AVMPLUS_SPARC +// Note: the linux #define provided by the compiler. +#ifdef linux // bugzilla 502369 +void sync_instruction_memory(caddr_t v, u_int len) +{ + caddr_t end = v + len; + caddr_t p = v; + while (p < end) { + asm("flush %0" : : "r" (p)); + p += 32; + } +} +#else +extern "C" void sync_instruction_memory(caddr_t v, u_int len); +#endif +#endif + +#if defined NANOJIT_IA32 || defined NANOJIT_X64 + // intel chips have dcache/icache interlock + void CodeAlloc::flushICache(void *start, size_t len) { + // Tell Valgrind that new code has been generated, and it must flush + // any translations it has for the memory range generated into. + (void)start; + (void)len; + VALGRIND_DISCARD_TRANSLATIONS(start, len); + } + +#elif defined NANOJIT_ARM && defined UNDER_CE + // On arm/winmo, just flush the whole icache. The + // WinCE docs indicate that this function actually ignores its + // 2nd and 3rd arguments, and wants them to be NULL. + void CodeAlloc::flushICache(void *, size_t) { + FlushInstructionCache(GetCurrentProcess(), NULL, NULL); + } + +#elif defined NANOJIT_ARM && defined DARWIN + void CodeAlloc::flushICache(void *, size_t) { + VMPI_debugBreak(); + } + +#elif defined AVMPLUS_MAC && defined NANOJIT_PPC + +# ifdef NANOJIT_64BIT + extern "C" void sys_icache_invalidate(const void*, size_t len); + extern "C" void sys_dcache_flush(const void*, size_t len); + + // mac 64bit requires 10.5 so use that api + void CodeAlloc::flushICache(void *start, size_t len) { + sys_dcache_flush(start, len); + sys_icache_invalidate(start, len); + } +# else + // mac ppc 32 could be 10.0 or later + // uses MakeDataExecutable() from Carbon api, OSUtils.h + // see http://developer.apple.com/documentation/Carbon/Reference/Memory_Manag_nt_Utilities/Reference/reference.html#//apple_ref/c/func/MakeDataExecutable + void CodeAlloc::flushICache(void *start, size_t len) { + MakeDataExecutable(start, len); + } +# endif + +#elif defined NANOJIT_ARM && defined VMCFG_SYMBIAN + void CodeAlloc::flushICache(void *ptr, size_t len) { + uint32_t start = (uint32_t)ptr; + uint32_t rangeEnd = start + len; + User::IMB_Range((TAny*)start, (TAny*)rangeEnd); + } + +#elif defined AVMPLUS_SPARC + // fixme: sync_instruction_memory is a solaris api, test for solaris not sparc + void CodeAlloc::flushICache(void *start, size_t len) { + sync_instruction_memory((char*)start, len); + } + +#elif defined NANOJIT_SH4 +#include <asm/cachectl.h> /* CACHEFLUSH_*, */ +#include <sys/syscall.h> /* __NR_cacheflush, */ + void CodeAlloc::flushICache(void *start, size_t len) { + syscall(__NR_cacheflush, start, len, CACHEFLUSH_D_WB | CACHEFLUSH_I); + } + +#elif defined(AVMPLUS_UNIX) && defined(NANOJIT_MIPS) + void CodeAlloc::flushICache(void *start, size_t len) { + // FIXME Use synci on MIPS32R2 + cacheflush((char *)start, len, BCACHE); + } + +#elif defined AVMPLUS_UNIX + #ifdef ANDROID + void CodeAlloc::flushICache(void *start, size_t len) { + cacheflush((int)start, (int)start + len, 0); + } + #else + // fixme: __clear_cache is a libgcc feature, test for libgcc or gcc + void CodeAlloc::flushICache(void *start, size_t len) { + __clear_cache((char*)start, (char*)start + len); + } + #endif +#endif // AVMPLUS_MAC && NANOJIT_PPC + + void CodeAlloc::addBlock(CodeList* &blocks, CodeList* b) { + NanoAssert(b->terminator != NULL); // should not be mucking with terminator blocks + b->next = blocks; + blocks = b; + } + + void CodeAlloc::addMem() { + void *mem = allocCodeChunk(bytesPerAlloc); // allocations never fail + totalAllocated += bytesPerAlloc; + NanoAssert(mem != NULL); // see allocCodeChunk contract in CodeAlloc.h + _nvprof("alloc page", uintptr_t(mem)>>12); + + CodeList* b = (CodeList*)mem; + b->lower = 0; + b->next = 0; + b->end = (NIns*) (uintptr_t(mem) + bytesPerAlloc - sizeofMinBlock); + b->isFree = true; + + // create a tiny terminator block, add to fragmented list, this way + // all other blocks have a valid block at b->higher + CodeList* terminator = b->higher; + b->terminator = terminator; + terminator->lower = b; + terminator->end = 0; // this is how we identify the terminator + terminator->isFree = false; + terminator->isExec = false; + terminator->terminator = 0; + debug_only(sanity_check();) + + // add terminator to heapblocks list so we can track whole blocks + terminator->next = heapblocks; + heapblocks = terminator; + + addBlock(availblocks, b); // add to free list + } + + CodeList* CodeAlloc::getBlock(NIns* start, NIns* end) { + CodeList* b = (CodeList*) (uintptr_t(start) - offsetof(CodeList, code)); + NanoAssert(b->end == end && b->next == 0); (void) end; + return b; + } + + CodeList* CodeAlloc::removeBlock(CodeList* &blocks) { + CodeList* b = blocks; + NanoAssert(b != NULL); + NanoAssert(b->terminator != NULL); // should not be mucking with terminator blocks + blocks = b->next; + b->next = 0; + return b; + } + + void CodeAlloc::add(CodeList* &blocks, NIns* start, NIns* end) { + addBlock(blocks, getBlock(start, end)); + } + + /** + * split a block by freeing the hole in the middle defined by [holeStart,holeEnd), + * and adding the used prefix and suffix parts to the blocks CodeList. + */ + void CodeAlloc::addRemainder(CodeList* &blocks, NIns* start, NIns* end, NIns* holeStart, NIns* holeEnd) { + NanoAssert(start < end && start <= holeStart && holeStart <= holeEnd && holeEnd <= end); + // shrink the hole by aligning holeStart forward and holeEnd backward + holeStart = (NIns*) ((uintptr_t(holeStart) + sizeof(NIns*)-1) & ~(sizeof(NIns*)-1)); + holeEnd = (NIns*) (uintptr_t(holeEnd) & ~(sizeof(NIns*)-1)); + // hole needs to be big enough for 2 headers + 1 block of free space (subtraction not used in check to avoid wraparound) + size_t minHole = headerSpaceFor(2) + blkSpaceFor(1); + if (uintptr_t(holeEnd) < minHole + uintptr_t(holeStart) ) { + // the hole is too small to make a new free block and a new used block. just keep + // the whole original block and don't free anything. + add(blocks, start, end); + } else if (holeStart == start && holeEnd == end) { + // totally empty block. free whole start-end range + this->free(start, end); + } else if (holeStart == start) { + // hole is lower-aligned with start, so just need one new block + // b1 b2 + CodeList* b1 = getBlock(start, end); + CodeList* b2 = (CodeList*) (uintptr_t(holeEnd) - offsetof(CodeList, code)); + b2->terminator = b1->terminator; + b2->isFree = false; + b2->next = 0; + b2->higher = b1->higher; + b2->lower = b1; + b2->higher->lower = b2; + b1->higher = b2; + debug_only(sanity_check();) + this->free(b1->start(), b1->end); + addBlock(blocks, b2); + } else if (holeEnd == end) { + // hole is right-aligned with end, just need one new block + // todo + NanoAssert(false); + } else { + // there's enough space left to split into three blocks (two new ones) + CodeList* b1 = getBlock(start, end); + CodeList* b2 = (CodeList*) (void*) holeStart; + CodeList* b3 = (CodeList*) (uintptr_t(holeEnd) - offsetof(CodeList, code)); + b1->higher = b2; + b2->lower = b1; + b2->higher = b3; + b2->isFree = false; // redundant, since we're about to free, but good hygiene + b2->terminator = b1->terminator; + b3->lower = b2; + b3->end = end; + b3->isFree = false; + b3->higher->lower = b3; + b3->terminator = b1->terminator; + b2->next = 0; + b3->next = 0; + debug_only(sanity_check();) + this->free(b2->start(), b2->end); + addBlock(blocks, b3); + addBlock(blocks, b1); + } + } + +#ifdef PERFM + // This method is used only for profiling purposes. + // See CodegenLIR::emitMD() in Tamarin for an example. + + size_t CodeAlloc::size(const CodeList* blocks) { + size_t size = 0; + for (const CodeList* b = blocks; b != 0; b = b->next) + size += int((uintptr_t)b->end - (uintptr_t)b); + return size; + } +#endif + + size_t CodeAlloc::size() { + return totalAllocated; + } + + // check that all block neighbors are correct + #ifdef _DEBUG + void CodeAlloc::sanity_check() { + for (CodeList* hb = heapblocks; hb != 0; hb = hb->next) { + NanoAssert(hb->higher == 0); + for (CodeList* b = hb->lower; b != 0; b = b->lower) { + NanoAssert(b->higher->lower == b); + } + } + for (CodeList* avail = this->availblocks; avail; avail = avail->next) { + NanoAssert(avail->isFree && avail->size() >= minAllocSize); + } + + #if CROSS_CHECK_FREE_LIST + for(CodeList* term = heapblocks; term; term = term->next) { + for(CodeList* hb = term->lower; hb; hb = hb->lower) { + if (hb->isFree && hb->size() >= minAllocSize) { + bool found_on_avail = false; + for (CodeList* avail = this->availblocks; !found_on_avail && avail; avail = avail->next) { + found_on_avail = avail == hb; + } + + NanoAssert(found_on_avail); + } + } + } + for (CodeList* avail = this->availblocks; avail; avail = avail->next) { + bool found_in_heapblocks = false; + for(CodeList* term = heapblocks; !found_in_heapblocks && term; term = term->next) { + for(CodeList* hb = term->lower; !found_in_heapblocks && hb; hb = hb->lower) { + found_in_heapblocks = hb == avail; + } + } + NanoAssert(found_in_heapblocks); + } + #endif /* CROSS_CHECK_FREE_LIST */ + } + #endif + + // Loop through a list of blocks marking the chunks executable. If we encounter + // multiple blocks in the same chunk, only the first block will cause the + // chunk to become executable, the other calls will no-op (isExec flag checked) + void CodeAlloc::markExec(CodeList* &blocks) { + for (CodeList *b = blocks; b != 0; b = b->next) { + markChunkExec(b->terminator); + } + } + + // Variant of markExec(CodeList*) that walks all heapblocks (i.e. chunks) marking + // each one executable. On systems where bytesPerAlloc is low (i.e. have lots + // of elements in the list) this can be expensive. + void CodeAlloc::markAllExec() { + for (CodeList* hb = heapblocks; hb != NULL; hb = hb->next) { + markChunkExec(hb); + } + } + + // make an entire chunk executable + void CodeAlloc::markChunkExec(CodeList* term) { + NanoAssert(term->terminator == NULL); + if (!term->isExec) { + term->isExec = true; + markCodeChunkExec(firstBlock(term), bytesPerAlloc); + } + } +} +#endif // FEATURE_NANOJIT |