// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*- // Copyright (c) 2000, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // --- // Author: Urs Holzle #include "config.h" #include #ifdef HAVE_FCNTL_H #include #endif #ifdef HAVE_INTTYPES_H #include #endif // We only need malloc.h for struct mallinfo. #ifdef HAVE_STRUCT_MALLINFO // Malloc can be in several places on older versions of OS X. # if defined(HAVE_MALLOC_H) # include # elif defined(HAVE_MALLOC_MALLOC_H) # include # elif defined(HAVE_SYS_MALLOC_H) # include # endif #endif #ifdef HAVE_PTHREAD #include #endif #include #include #include #ifdef HAVE_MMAP #include #endif #include #include #ifdef HAVE_UNISTD_H #include #endif #include #include #include #include "addressmap-inl.h" #include "base/commandlineflags.h" #include "base/googleinit.h" #include "base/logging.h" #include "base/spinlock.h" #include "malloc_hook-inl.h" #include "symbolize.h" // NOTE: due to #define below, tcmalloc.cc will omit tc_XXX // definitions. So that debug implementations can be defined // instead. We're going to use do_malloc, do_free and other do_XXX // functions that are defined in tcmalloc.cc for actual memory // management #define TCMALLOC_USING_DEBUGALLOCATION #include "tcmalloc.cc" // __THROW is defined in glibc systems. It means, counter-intuitively, // "This function will never throw an exception." It's an optional // optimization tool, but we may need to use it to match glibc prototypes. #ifndef __THROW // I guess we're not on a glibc system # define __THROW // __THROW is just an optimization, so ok to make it "" #endif // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old // form of the name instead. #ifndef MAP_ANONYMOUS # define MAP_ANONYMOUS MAP_ANON #endif // ========================================================================= // DEFINE_bool(malloctrace, EnvToBool("TCMALLOC_TRACE", false), "Enables memory (de)allocation tracing to /tmp/google.alloc."); #ifdef HAVE_MMAP DEFINE_bool(malloc_page_fence, EnvToBool("TCMALLOC_PAGE_FENCE", false), "Enables putting of memory allocations at page boundaries " "with a guard page following the allocation (to catch buffer " "overruns right when they happen)."); DEFINE_bool(malloc_page_fence_never_reclaim, EnvToBool("TCMALLOC_PAGE_FENCE_NEVER_RECLAIM", false), "Enables making the virtual address space inaccessible " "upon a deallocation instead of returning it and reusing later."); #else DEFINE_bool(malloc_page_fence, false, "Not usable (requires mmap)"); DEFINE_bool(malloc_page_fence_never_reclaim, false, "Not usable (required mmap)"); #endif DEFINE_bool(malloc_reclaim_memory, EnvToBool("TCMALLOC_RECLAIM_MEMORY", true), "If set to false, we never return memory to malloc " "when an object is deallocated. This ensures that all " "heap object addresses are unique."); DEFINE_int32(max_free_queue_size, EnvToInt("TCMALLOC_MAX_FREE_QUEUE_SIZE", 10*1024*1024), "If greater than 0, keep freed blocks in a queue instead of " "releasing them to the allocator immediately. Release them when " "the total size of all blocks in the queue would otherwise exceed " "this limit."); DEFINE_bool(symbolize_stacktrace, EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true), "Symbolize the stack trace when provided (on some error exits)"); // If we are LD_PRELOAD-ed against a non-pthreads app, then // pthread_once won't be defined. We declare it here, for that // case (with weak linkage) which will cause the non-definition to // resolve to NULL. We can then check for NULL or not in Instance. extern "C" int pthread_once(pthread_once_t *, void (*)(void)) ATTRIBUTE_WEAK; // ========================================================================= // // A safe version of printf() that does not do any allocation and // uses very little stack space. static void TracePrintf(int fd, const char *fmt, ...) __attribute__ ((__format__ (__printf__, 2, 3))); // Round "value" up to next "alignment" boundary. // Requires that "alignment" be a power of two. static intptr_t RoundUp(intptr_t value, intptr_t alignment) { return (value + alignment - 1) & ~(alignment - 1); } // ========================================================================= // class MallocBlock; // A circular buffer to hold freed blocks of memory. MallocBlock::Deallocate // (below) pushes blocks into this queue instead of returning them to the // underlying allocator immediately. See MallocBlock::Deallocate for more // information. // // We can't use an STL class for this because we need to be careful not to // perform any heap de-allocations in any of the code in this class, since the // code in MallocBlock::Deallocate is not re-entrant. template class FreeQueue { public: FreeQueue() : q_front_(0), q_back_(0) {} bool Full() { return (q_front_ + 1) % kFreeQueueSize == q_back_; } void Push(const QueueEntry& block) { q_[q_front_] = block; q_front_ = (q_front_ + 1) % kFreeQueueSize; } QueueEntry Pop() { RAW_CHECK(q_back_ != q_front_, "Queue is empty"); const QueueEntry& ret = q_[q_back_]; q_back_ = (q_back_ + 1) % kFreeQueueSize; return ret; } size_t size() const { return (q_front_ - q_back_ + kFreeQueueSize) % kFreeQueueSize; } private: // Maximum number of blocks kept in the free queue before being freed. static const int kFreeQueueSize = 1024; QueueEntry q_[kFreeQueueSize]; int q_front_; int q_back_; }; struct MallocBlockQueueEntry { MallocBlockQueueEntry() : block(NULL), size(0), num_deleter_pcs(0), deleter_threadid(0) {} MallocBlockQueueEntry(MallocBlock* b, size_t s) : block(b), size(s) { if (FLAGS_max_free_queue_size != 0 && b != NULL) { // Adjust the number of frames to skip (4) if you change the // location of this call. num_deleter_pcs = MallocHook::GetCallerStackTrace( deleter_pcs, sizeof(deleter_pcs) / sizeof(deleter_pcs[0]), 4); deleter_threadid = pthread_self(); } else { num_deleter_pcs = 0; // Zero is an illegal pthread id by my reading of the pthread // implementation: deleter_threadid = 0; } } MallocBlock* block; size_t size; // When deleted and put in the free queue, we (flag-controlled) // record the stack so that if corruption is later found, we can // print the deleter's stack. (These three vars add 144 bytes of // overhead under the LP64 data model.) void* deleter_pcs[16]; int num_deleter_pcs; pthread_t deleter_threadid; }; class MallocBlock { public: // allocation type constants // Different allocation types we distinguish. // Note: The lower 4 bits are not random: we index kAllocName array // by these values masked with kAllocTypeMask; // the rest are "random" magic bits to help catch memory corruption. static const int kMallocType = 0xEFCDAB90; static const int kNewType = 0xFEBADC81; static const int kArrayNewType = 0xBCEADF72; private: // constants // A mask used on alloc types above to get to 0, 1, 2 static const int kAllocTypeMask = 0x3; // An additional bit to set in AllocType constants // to mark now deallocated regions. static const int kDeallocatedTypeBit = 0x4; // For better memory debugging, we initialize all storage to known // values, and overwrite the storage when it's deallocated: // Byte that fills uninitialized storage. static const int kMagicUninitializedByte = 0xAB; // Byte that fills deallocated storage. // NOTE: tcmalloc.cc depends on the value of kMagicDeletedByte // to work around a bug in the pthread library. static const int kMagicDeletedByte = 0xCD; // A size_t (type of alloc_type_ below) in a deallocated storage // filled with kMagicDeletedByte. static const size_t kMagicDeletedSizeT = 0xCDCDCDCD | (((size_t)0xCDCDCDCD << 16) << 16); // Initializer works for 32 and 64 bit size_ts; // "<< 16 << 16" is to fool gcc from issuing a warning // when size_ts are 32 bits. // NOTE: on Linux, you can enable malloc debugging support in libc by // setting the environment variable MALLOC_CHECK_ to 1 before you // start the program (see man malloc). // We use either do_malloc or mmap to make the actual allocation. In // order to remember which one of the two was used for any block, we store an // appropriate magic word next to the block. static const size_t kMagicMalloc = 0xDEADBEEF; static const size_t kMagicMMap = 0xABCDEFAB; // This array will be filled with 0xCD, for use with memcmp. static unsigned char kMagicDeletedBuffer[1024]; static pthread_once_t deleted_buffer_initialized_; static bool deleted_buffer_initialized_no_pthreads_; private: // data layout // The four fields size1_,offset_,magic1_,alloc_type_ // should together occupy a multiple of 16 bytes. (At the // moment, sizeof(size_t) == 4 or 8 depending on piii vs // k8, and 4 of those sum to 16 or 32 bytes). // This, combined with do_malloc's alignment guarantees, // ensures that SSE types can be stored into the returned // block, at &size2_. size_t size1_; size_t offset_; // normally 0 unless memaligned memory // see comments in memalign() and FromRawPointer(). size_t magic1_; size_t alloc_type_; // here comes the actual data (variable length) // ... // then come the size2_ and magic2_, or a full page of mprotect-ed memory // if the malloc_page_fence feature is enabled. size_t size2_; size_t magic2_; private: // static data and helpers // Allocation map: stores the allocation type for each allocated object, // or the type or'ed with kDeallocatedTypeBit // for each formerly allocated object. typedef AddressMap AllocMap; static AllocMap* alloc_map_; // This protects alloc_map_ and consistent state of metadata // for each still-allocated object in it. // We use spin locks instead of pthread_mutex_t locks // to prevent crashes via calls to pthread_mutex_(un)lock // for the (de)allocations coming from pthreads initialization itself. static SpinLock alloc_map_lock_; // A queue of freed blocks. Instead of releasing blocks to the allocator // immediately, we put them in a queue, freeing them only when necessary // to keep the total size of all the freed blocks below the limit set by // FLAGS_max_free_queue_size. static FreeQueue* free_queue_; static size_t free_queue_size_; // total size of blocks in free_queue_ // protects free_queue_ and free_queue_size_ static SpinLock free_queue_lock_; // Names of allocation types (kMallocType, kNewType, kArrayNewType) static const char* const kAllocName[]; // Names of corresponding deallocation types static const char* const kDeallocName[]; static const char* AllocName(int type) { return kAllocName[type & kAllocTypeMask]; } static const char* DeallocName(int type) { return kDeallocName[type & kAllocTypeMask]; } private: // helper accessors bool IsMMapped() const { return kMagicMMap == magic1_; } bool IsValidMagicValue(size_t value) const { return kMagicMMap == value || kMagicMalloc == value; } static size_t real_malloced_size(size_t size) { return size + sizeof(MallocBlock); } /* * Here we assume size of page is kMinAlign aligned, * so if size is MALLOC_ALIGNMENT aligned too, then we could * guarantee return address is also kMinAlign aligned, because * mmap return address at nearby page boundary on Linux. */ static size_t real_mmapped_size(size_t size) { size_t tmp = size + MallocBlock::data_offset(); tmp = RoundUp(tmp, kMinAlign); return tmp; } size_t real_size() { return IsMMapped() ? real_mmapped_size(size1_) : real_malloced_size(size1_); } // NOTE: if the block is mmapped (that is, we're using the // malloc_page_fence option) then there's no size2 or magic2 // (instead, the guard page begins where size2 would be). size_t* size2_addr() { return (size_t*)((char*)&size2_ + size1_); } const size_t* size2_addr() const { return (const size_t*)((char*)&size2_ + size1_); } size_t* magic2_addr() { return (size_t*)(size2_addr() + 1); } const size_t* magic2_addr() const { return (const size_t*)(size2_addr() + 1); } private: // other helpers void Initialize(size_t size, int type) { RAW_CHECK(IsValidMagicValue(magic1_), ""); // record us as allocated in the map alloc_map_lock_.Lock(); if (!alloc_map_) { void* p = do_malloc(sizeof(AllocMap)); alloc_map_ = new(p) AllocMap(do_malloc, do_free); } alloc_map_->Insert(data_addr(), type); // initialize us size1_ = size; offset_ = 0; alloc_type_ = type; if (!IsMMapped()) { bit_store(magic2_addr(), &magic1_); bit_store(size2_addr(), &size); } alloc_map_lock_.Unlock(); memset(data_addr(), kMagicUninitializedByte, size); if (!IsMMapped()) { RAW_CHECK(memcmp(&size1_, size2_addr(), sizeof(size1_)) == 0, "should hold"); RAW_CHECK(memcmp(&magic1_, magic2_addr(), sizeof(magic1_)) == 0, "should hold"); } } size_t CheckAndClear(int type, size_t given_size) { alloc_map_lock_.Lock(); CheckLocked(type); if (!IsMMapped()) { RAW_CHECK(memcmp(&size1_, size2_addr(), sizeof(size1_)) == 0, "should hold"); } // record us as deallocated in the map alloc_map_->Insert(data_addr(), type | kDeallocatedTypeBit); alloc_map_lock_.Unlock(); // clear us const size_t size = real_size(); RAW_CHECK(!given_size || given_size == size1_, "right size must be passed to sized delete"); memset(this, kMagicDeletedByte, size); return size; } void CheckLocked(int type) const { int map_type = 0; const int* found_type = alloc_map_ != NULL ? alloc_map_->Find(data_addr()) : NULL; if (found_type == NULL) { RAW_LOG(FATAL, "memory allocation bug: object at %p " "has never been allocated", data_addr()); } else { map_type = *found_type; } if ((map_type & kDeallocatedTypeBit) != 0) { RAW_LOG(FATAL, "memory allocation bug: object at %p " "has been already deallocated (it was allocated with %s)", data_addr(), AllocName(map_type & ~kDeallocatedTypeBit)); } if (alloc_type_ == kMagicDeletedSizeT) { RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " "has been corrupted; or else the object has been already " "deallocated and our memory map has been corrupted", data_addr()); } if (!IsValidMagicValue(magic1_)) { RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " "has been corrupted; " "or else our memory map has been corrupted and this is a " "deallocation for not (currently) heap-allocated object", data_addr()); } if (!IsMMapped()) { if (memcmp(&size1_, size2_addr(), sizeof(size1_))) { RAW_LOG(FATAL, "memory stomping bug: a word after object at %p " "has been corrupted", data_addr()); } size_t addr; bit_store(&addr, magic2_addr()); if (!IsValidMagicValue(addr)) { RAW_LOG(FATAL, "memory stomping bug: a word after object at %p " "has been corrupted", data_addr()); } } if (alloc_type_ != type) { if ((alloc_type_ != MallocBlock::kMallocType) && (alloc_type_ != MallocBlock::kNewType) && (alloc_type_ != MallocBlock::kArrayNewType)) { RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " "has been corrupted", data_addr()); } RAW_LOG(FATAL, "memory allocation/deallocation mismatch at %p: " "allocated with %s being deallocated with %s", data_addr(), AllocName(alloc_type_), DeallocName(type)); } if (alloc_type_ != map_type) { RAW_LOG(FATAL, "memory stomping bug: our memory map has been corrupted : " "allocation at %p made with %s " "is recorded in the map to be made with %s", data_addr(), AllocName(alloc_type_), AllocName(map_type)); } } public: // public accessors void* data_addr() { return (void*)&size2_; } const void* data_addr() const { return (const void*)&size2_; } static size_t data_offset() { return OFFSETOF_MEMBER(MallocBlock, size2_); } size_t data_size() const { return size1_; } void set_offset(int offset) { this->offset_ = offset; } public: // our main interface static MallocBlock* Allocate(size_t size, int type) { // Prevent an integer overflow / crash with large allocation sizes. // TODO - Note that for a e.g. 64-bit size_t, max_size_t may not actually // be the maximum value, depending on how the compiler treats ~0. The worst // practical effect is that allocations are limited to 4Gb or so, even if // the address space could take more. static size_t max_size_t = ~0; if (size > max_size_t - sizeof(MallocBlock)) { RAW_LOG(ERROR, "Massive size passed to malloc: %" PRIuS "", size); return NULL; } MallocBlock* b = NULL; const bool use_malloc_page_fence = FLAGS_malloc_page_fence; #ifdef HAVE_MMAP if (use_malloc_page_fence) { // Put the block towards the end of the page and make the next page // inaccessible. This will catch buffer overrun right when it happens. size_t sz = real_mmapped_size(size); int pagesize = getpagesize(); int num_pages = (sz + pagesize - 1) / pagesize + 1; char* p = (char*) mmap(NULL, num_pages * pagesize, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); if (p == MAP_FAILED) { // If the allocation fails, abort rather than returning NULL to // malloc. This is because in most cases, the program will run out // of memory in this mode due to tremendous amount of wastage. There // is no point in propagating the error elsewhere. RAW_LOG(FATAL, "Out of memory: possibly due to page fence overhead: %s", strerror(errno)); } // Mark the page after the block inaccessible if (mprotect(p + (num_pages - 1) * pagesize, pagesize, PROT_NONE)) { RAW_LOG(FATAL, "Guard page setup failed: %s", strerror(errno)); } b = (MallocBlock*) (p + (num_pages - 1) * pagesize - sz); } else { b = (MallocBlock*) do_malloc(real_malloced_size(size)); } #else b = (MallocBlock*) do_malloc(real_malloced_size(size)); #endif // It would be nice to output a diagnostic on allocation failure // here, but logging (other than FATAL) requires allocating // memory, which could trigger a nasty recursion. Instead, preserve // malloc semantics and return NULL on failure. if (b != NULL) { b->magic1_ = use_malloc_page_fence ? kMagicMMap : kMagicMalloc; b->Initialize(size, type); } return b; } void Deallocate(int type, size_t given_size) { if (IsMMapped()) { // have to do this before CheckAndClear #ifdef HAVE_MMAP int size = CheckAndClear(type, given_size); int pagesize = getpagesize(); int num_pages = (size + pagesize - 1) / pagesize + 1; char* p = (char*) this; if (FLAGS_malloc_page_fence_never_reclaim || !FLAGS_malloc_reclaim_memory) { mprotect(p - (num_pages - 1) * pagesize + size, num_pages * pagesize, PROT_NONE); } else { munmap(p - (num_pages - 1) * pagesize + size, num_pages * pagesize); } #endif } else { const size_t size = CheckAndClear(type, given_size); if (FLAGS_malloc_reclaim_memory) { // Instead of freeing the block immediately, push it onto a queue of // recently freed blocks. Free only enough blocks to keep from // exceeding the capacity of the queue or causing the total amount of // un-released memory in the queue from exceeding // FLAGS_max_free_queue_size. ProcessFreeQueue(this, size, FLAGS_max_free_queue_size); } } } static size_t FreeQueueSize() { SpinLockHolder l(&free_queue_lock_); return free_queue_size_; } static void ProcessFreeQueue(MallocBlock* b, size_t size, int max_free_queue_size) { // MallocBlockQueueEntry are about 144 in size, so we can only // use a small array of them on the stack. MallocBlockQueueEntry entries[4]; int num_entries = 0; MallocBlockQueueEntry new_entry(b, size); free_queue_lock_.Lock(); if (free_queue_ == NULL) free_queue_ = new FreeQueue; RAW_CHECK(!free_queue_->Full(), "Free queue mustn't be full!"); if (b != NULL) { free_queue_size_ += size + sizeof(MallocBlockQueueEntry); free_queue_->Push(new_entry); } // Free blocks until the total size of unfreed blocks no longer exceeds // max_free_queue_size, and the free queue has at least one free // space in it. while (free_queue_size_ > max_free_queue_size || free_queue_->Full()) { RAW_CHECK(num_entries < arraysize(entries), "entries array overflow"); entries[num_entries] = free_queue_->Pop(); free_queue_size_ -= entries[num_entries].size + sizeof(MallocBlockQueueEntry); num_entries++; if (num_entries == arraysize(entries)) { // The queue will not be full at this point, so it is ok to // release the lock. The queue may still contain more than // max_free_queue_size, but this is not a strict invariant. free_queue_lock_.Unlock(); for (int i = 0; i < num_entries; i++) { CheckForDanglingWrites(entries[i]); do_free(entries[i].block); } num_entries = 0; free_queue_lock_.Lock(); } } free_queue_lock_.Unlock(); for (int i = 0; i < num_entries; i++) { CheckForDanglingWrites(entries[i]); do_free(entries[i].block); } } static void InitDeletedBuffer() { memset(kMagicDeletedBuffer, kMagicDeletedByte, sizeof(kMagicDeletedBuffer)); deleted_buffer_initialized_no_pthreads_ = true; } static void CheckForDanglingWrites(const MallocBlockQueueEntry& queue_entry) { // Initialize the buffer if necessary. if (pthread_once) pthread_once(&deleted_buffer_initialized_, &InitDeletedBuffer); if (!deleted_buffer_initialized_no_pthreads_) { // This will be the case on systems that don't link in pthreads, // including on FreeBSD where pthread_once has a non-zero address // (but doesn't do anything) even when pthreads isn't linked in. InitDeletedBuffer(); } const unsigned char* p = reinterpret_cast(queue_entry.block); static const size_t size_of_buffer = sizeof(kMagicDeletedBuffer); const size_t size = queue_entry.size; const size_t buffers = size / size_of_buffer; const size_t remainder = size % size_of_buffer; size_t buffer_idx; for (buffer_idx = 0; buffer_idx < buffers; ++buffer_idx) { CheckForCorruptedBuffer(queue_entry, buffer_idx, p, size_of_buffer); p += size_of_buffer; } CheckForCorruptedBuffer(queue_entry, buffer_idx, p, remainder); } static void CheckForCorruptedBuffer(const MallocBlockQueueEntry& queue_entry, size_t buffer_idx, const unsigned char* buffer, size_t size_of_buffer) { if (memcmp(buffer, kMagicDeletedBuffer, size_of_buffer) == 0) { return; } RAW_LOG(ERROR, "Found a corrupted memory buffer in MallocBlock (may be offset " "from user ptr): buffer index: %zd, buffer ptr: %p, size of " "buffer: %zd", buffer_idx, buffer, size_of_buffer); // The magic deleted buffer should only be 1024 bytes, but in case // this changes, let's put an upper limit on the number of debug // lines we'll output: if (size_of_buffer <= 1024) { for (int i = 0; i < size_of_buffer; ++i) { if (buffer[i] != kMagicDeletedByte) { RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0x%02x).", i, buffer[i], kMagicDeletedByte); } } } else { RAW_LOG(ERROR, "Buffer too large to print corruption."); } const MallocBlock* b = queue_entry.block; const size_t size = queue_entry.size; if (queue_entry.num_deleter_pcs > 0) { TracePrintf(STDERR_FILENO, "Deleted by thread %p\n", reinterpret_cast( PRINTABLE_PTHREAD(queue_entry.deleter_threadid))); // We don't want to allocate or deallocate memory here, so we use // placement-new. It's ok that we don't destroy this, since we're // just going to error-exit below anyway. Union is for alignment. union { void* alignment; char buf[sizeof(SymbolTable)]; } tablebuf; SymbolTable* symbolization_table = new (tablebuf.buf) SymbolTable; for (int i = 0; i < queue_entry.num_deleter_pcs; i++) { // Symbolizes the previous address of pc because pc may be in the // next function. This may happen when the function ends with // a call to a function annotated noreturn (e.g. CHECK). char *pc = reinterpret_cast(queue_entry.deleter_pcs[i]); symbolization_table->Add(pc - 1); } if (FLAGS_symbolize_stacktrace) symbolization_table->Symbolize(); for (int i = 0; i < queue_entry.num_deleter_pcs; i++) { char *pc = reinterpret_cast(queue_entry.deleter_pcs[i]); TracePrintf(STDERR_FILENO, " @ %p %s\n", pc, symbolization_table->GetSymbol(pc - 1)); } } else { RAW_LOG(ERROR, "Skipping the printing of the deleter's stack! Its stack was " "not found; either the corruption occurred too early in " "execution to obtain a stack trace or --max_free_queue_size was " "set to 0."); } RAW_LOG(FATAL, "Memory was written to after being freed. MallocBlock: %p, user " "ptr: %p, size: %zd. If you can't find the source of the error, " "try using ASan (http://code.google.com/p/address-sanitizer/), " "Valgrind, or Purify, or study the " "output of the deleter's stack printed above.", b, b->data_addr(), size); } static MallocBlock* FromRawPointer(void* p) { const size_t data_offset = MallocBlock::data_offset(); // Find the header just before client's memory. MallocBlock *mb = reinterpret_cast( reinterpret_cast(p) - data_offset); // If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer. if (mb->alloc_type_ == kMagicDeletedSizeT) { RAW_LOG(FATAL, "memory allocation bug: object at %p has been already" " deallocated; or else a word before the object has been" " corrupted (memory stomping bug)", p); } // If mb->offset_ is zero (common case), mb is the real header. // If mb->offset_ is non-zero, this block was allocated by debug // memallign implementation, and mb->offset_ is the distance // backwards to the real header from mb, which is a fake header. if (mb->offset_ == 0) { return mb; } MallocBlock *main_block = reinterpret_cast( reinterpret_cast(mb) - mb->offset_); if (main_block->offset_ != 0) { RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted." " Need 0 but got %x", (unsigned)(main_block->offset_)); } if (main_block >= p) { RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted." " Detected main_block address overflow: %x", (unsigned)(mb->offset_)); } if (main_block->size2_addr() < p) { RAW_LOG(FATAL, "memory corruption bug: offset_ field is corrupted." " It points below it's own main_block: %x", (unsigned)(mb->offset_)); } return main_block; } static const MallocBlock* FromRawPointer(const void* p) { // const-safe version: we just cast about return FromRawPointer(const_cast(p)); } void Check(int type) const { alloc_map_lock_.Lock(); CheckLocked(type); alloc_map_lock_.Unlock(); } static bool CheckEverything() { alloc_map_lock_.Lock(); if (alloc_map_ != NULL) alloc_map_->Iterate(CheckCallback, 0); alloc_map_lock_.Unlock(); return true; // if we get here, we're okay } static bool MemoryStats(int* blocks, size_t* total, int histogram[kMallocHistogramSize]) { memset(histogram, 0, kMallocHistogramSize * sizeof(int)); alloc_map_lock_.Lock(); stats_blocks_ = 0; stats_total_ = 0; stats_histogram_ = histogram; if (alloc_map_ != NULL) alloc_map_->Iterate(StatsCallback, 0); *blocks = stats_blocks_; *total = stats_total_; alloc_map_lock_.Unlock(); return true; } private: // helpers for CheckEverything and MemoryStats static void CheckCallback(const void* ptr, int* type, int dummy) { if ((*type & kDeallocatedTypeBit) == 0) { FromRawPointer(ptr)->CheckLocked(*type); } } // Accumulation variables for StatsCallback protected by alloc_map_lock_ static int stats_blocks_; static size_t stats_total_; static int* stats_histogram_; static void StatsCallback(const void* ptr, int* type, int dummy) { if ((*type & kDeallocatedTypeBit) == 0) { const MallocBlock* b = FromRawPointer(ptr); b->CheckLocked(*type); ++stats_blocks_; size_t mysize = b->size1_; int entry = 0; stats_total_ += mysize; while (mysize) { ++entry; mysize >>= 1; } RAW_CHECK(entry < kMallocHistogramSize, "kMallocHistogramSize should be at least as large as log2 " "of the maximum process memory size"); stats_histogram_[entry] += 1; } } }; void DanglingWriteChecker() { // Clear out the remaining free queue to check for dangling writes. MallocBlock::ProcessFreeQueue(NULL, 0, 0); } // ========================================================================= // const size_t MallocBlock::kMagicMalloc; const size_t MallocBlock::kMagicMMap; MallocBlock::AllocMap* MallocBlock::alloc_map_ = NULL; SpinLock MallocBlock::alloc_map_lock_(SpinLock::LINKER_INITIALIZED); FreeQueue* MallocBlock::free_queue_ = NULL; size_t MallocBlock::free_queue_size_ = 0; SpinLock MallocBlock::free_queue_lock_(SpinLock::LINKER_INITIALIZED); unsigned char MallocBlock::kMagicDeletedBuffer[1024]; pthread_once_t MallocBlock::deleted_buffer_initialized_ = PTHREAD_ONCE_INIT; bool MallocBlock::deleted_buffer_initialized_no_pthreads_ = false; const char* const MallocBlock::kAllocName[] = { "malloc", "new", "new []", NULL, }; const char* const MallocBlock::kDeallocName[] = { "free", "delete", "delete []", NULL, }; int MallocBlock::stats_blocks_; size_t MallocBlock::stats_total_; int* MallocBlock::stats_histogram_; // ========================================================================= // // The following cut-down version of printf() avoids // using stdio or ostreams. // This is to guarantee no recursive calls into // the allocator and to bound the stack space consumed. (The pthread // manager thread in linuxthreads has a very small stack, // so fprintf can't be called.) static void TracePrintf(int fd, const char *fmt, ...) { char buf[64]; int i = 0; va_list ap; va_start(ap, fmt); const char *p = fmt; char numbuf[25]; if (fd < 0) { va_end(ap); return; } numbuf[sizeof(numbuf)-1] = 0; while (*p != '\0') { // until end of format string char *s = &numbuf[sizeof(numbuf)-1]; if (p[0] == '%' && p[1] != 0) { // handle % formats int64 l = 0; unsigned long base = 0; if (*++p == 's') { // %s s = va_arg(ap, char *); } else if (*p == 'l' && p[1] == 'd') { // %ld l = va_arg(ap, long); base = 10; p++; } else if (*p == 'l' && p[1] == 'u') { // %lu l = va_arg(ap, unsigned long); base = 10; p++; } else if (*p == 'z' && p[1] == 'u') { // %zu l = va_arg(ap, size_t); base = 10; p++; } else if (*p == 'u') { // %u l = va_arg(ap, unsigned int); base = 10; } else if (*p == 'd') { // %d l = va_arg(ap, int); base = 10; } else if (*p == 'p') { // %p l = va_arg(ap, intptr_t); base = 16; } else { write(STDERR_FILENO, "Unimplemented TracePrintf format\n", 33); write(STDERR_FILENO, p, 2); write(STDERR_FILENO, "\n", 1); abort(); } p++; if (base != 0) { bool minus = (l < 0 && base == 10); uint64 ul = minus? -l : l; do { *--s = "0123456789abcdef"[ul % base]; ul /= base; } while (ul != 0); if (base == 16) { *--s = 'x'; *--s = '0'; } else if (minus) { *--s = '-'; } } } else { // handle normal characters *--s = *p++; } while (*s != 0) { if (i == sizeof(buf)) { write(fd, buf, i); i = 0; } buf[i++] = *s++; } } if (i != 0) { write(fd, buf, i); } va_end(ap); } // Return the file descriptor we're writing a log to static int TraceFd() { static int trace_fd = -1; if (trace_fd == -1) { // Open the trace file on the first call const char *val = getenv("TCMALLOC_TRACE_FILE"); bool fallback_to_stderr = false; if (!val) { val = "/tmp/google.alloc"; fallback_to_stderr = true; } trace_fd = open(val, O_CREAT|O_TRUNC|O_WRONLY, 0666); if (trace_fd == -1) { if (fallback_to_stderr) { trace_fd = 2; TracePrintf(trace_fd, "Can't open %s. Logging to stderr.\n", val); } else { TracePrintf(2, "Can't open %s. Logging disabled.\n", val); } } // Add a header to the log. TracePrintf(trace_fd, "Trace started: %lu\n", static_cast(time(NULL))); TracePrintf(trace_fd, "func\tsize\tptr\tthread_id\tstack pcs for tools/symbolize\n"); } return trace_fd; } // Print the hex stack dump on a single line. PCs are separated by tabs. static void TraceStack(void) { void *pcs[16]; int n = GetStackTrace(pcs, sizeof(pcs)/sizeof(pcs[0]), 0); for (int i = 0; i != n; i++) { TracePrintf(TraceFd(), "\t%p", pcs[i]); } } // This protects MALLOC_TRACE, to make sure its info is atomically written. static SpinLock malloc_trace_lock(SpinLock::LINKER_INITIALIZED); #define MALLOC_TRACE(name, size, addr) \ do { \ if (FLAGS_malloctrace) { \ SpinLockHolder l(&malloc_trace_lock); \ TracePrintf(TraceFd(), "%s\t%" PRIuS "\t%p\t%" GPRIuPTHREAD, \ name, size, addr, PRINTABLE_PTHREAD(pthread_self())); \ TraceStack(); \ TracePrintf(TraceFd(), "\n"); \ } \ } while (0) // ========================================================================= // // Write the characters buf[0, ..., size-1] to // the malloc trace buffer. // This function is intended for debugging, // and is not declared in any header file. // You must insert a declaration of it by hand when you need // to use it. void __malloctrace_write(const char *buf, size_t size) { if (FLAGS_malloctrace) { write(TraceFd(), buf, size); } } // ========================================================================= // // General debug allocation/deallocation static inline void* DebugAllocate(size_t size, int type) { MallocBlock* ptr = MallocBlock::Allocate(size, type); if (ptr == NULL) return NULL; MALLOC_TRACE("malloc", size, ptr->data_addr()); return ptr->data_addr(); } static inline void DebugDeallocate(void* ptr, int type, size_t given_size) { MALLOC_TRACE("free", (ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0), ptr); if (ptr) MallocBlock::FromRawPointer(ptr)->Deallocate(type, given_size); } // ========================================================================= // // The following functions may be called via MallocExtension::instance() // for memory verification and statistics. class DebugMallocImplementation : public TCMallocImplementation { public: virtual bool GetNumericProperty(const char* name, size_t* value) { bool result = TCMallocImplementation::GetNumericProperty(name, value); if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) { // Subtract bytes kept in the free queue size_t qsize = MallocBlock::FreeQueueSize(); if (*value >= qsize) { *value -= qsize; } } return result; } virtual bool VerifyNewMemory(const void* p) { if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType); return true; } virtual bool VerifyArrayNewMemory(const void* p) { if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType); return true; } virtual bool VerifyMallocMemory(const void* p) { if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType); return true; } virtual bool VerifyAllMemory() { return MallocBlock::CheckEverything(); } virtual bool MallocMemoryStats(int* blocks, size_t* total, int histogram[kMallocHistogramSize]) { return MallocBlock::MemoryStats(blocks, total, histogram); } virtual size_t GetEstimatedAllocatedSize(size_t size) { return size; } virtual size_t GetAllocatedSize(const void* p) { if (p) { RAW_CHECK(GetOwnership(p) != MallocExtension::kNotOwned, "ptr not allocated by tcmalloc"); return MallocBlock::FromRawPointer(p)->data_size(); } return 0; } virtual MallocExtension::Ownership GetOwnership(const void* p) { if (!p) { // nobody owns NULL return MallocExtension::kNotOwned; } // FIXME: note that correct GetOwnership should not touch memory // that is not owned by tcmalloc. Main implementation is using // pagemap to discover if page in question is owned by us or // not. But pagemap only has marks for first and last page of // spans. Note that if p was returned out of our memalign with // big alignment, then it will point outside of marked pages. Also // note that FromRawPointer call below requires touching memory // before pointer in order to handle memalign-ed chunks // (offset_). This leaves us with two options: // // * do FromRawPointer first and have possibility of crashing if // we're given not owned pointer // // * return incorrect ownership for those large memalign chunks // // I've decided to choose later, which appears to happen rarer and // therefore is arguably a lesser evil MallocExtension::Ownership rv = TCMallocImplementation::GetOwnership(p); if (rv != MallocExtension::kOwned) { return rv; } const MallocBlock* mb = MallocBlock::FromRawPointer(p); return TCMallocImplementation::GetOwnership(mb); } virtual void GetFreeListSizes(vector* v) { static const char* kDebugFreeQueue = "debug.free_queue"; TCMallocImplementation::GetFreeListSizes(v); MallocExtension::FreeListInfo i; i.type = kDebugFreeQueue; i.min_object_size = 0; i.max_object_size = numeric_limits::max(); i.total_bytes_free = MallocBlock::FreeQueueSize(); v->push_back(i); } }; static union { char chars[sizeof(DebugMallocImplementation)]; void *ptr; } debug_malloc_implementation_space; REGISTER_MODULE_INITIALIZER(debugallocation, { #if (__cplusplus >= 201103L) static_assert(alignof(decltype(debug_malloc_implementation_space)) >= alignof(DebugMallocImplementation), "DebugMallocImplementation is expected to need just word alignment"); #endif // Either we or valgrind will control memory management. We // register our extension if we're the winner. Otherwise let // Valgrind use its own malloc (so don't register our extension). if (!RunningOnValgrind()) { DebugMallocImplementation *impl = new (debug_malloc_implementation_space.chars) DebugMallocImplementation(); MallocExtension::Register(impl); } }); REGISTER_MODULE_DESTRUCTOR(debugallocation, { if (!RunningOnValgrind()) { // When the program exits, check all blocks still in the free // queue for corruption. DanglingWriteChecker(); } }); // ========================================================================= // struct debug_alloc_retry_data { size_t size; int new_type; }; static void *retry_debug_allocate(void *arg) { debug_alloc_retry_data *data = static_cast(arg); return DebugAllocate(data->size, data->new_type); } // This is mostly the same a cpp_alloc in tcmalloc.cc. // TODO(csilvers): change Allocate() above to call cpp_alloc, so we // don't have to reproduce the logic here. To make tc_new_mode work // properly, I think we'll need to separate out the logic of throwing // from the logic of calling the new-handler. inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) { void* p = DebugAllocate(size, new_type); if (p != NULL) { return p; } struct debug_alloc_retry_data data; data.size = size; data.new_type = new_type; return handle_oom(retry_debug_allocate, &data, true, nothrow); } inline void* do_debug_malloc_or_debug_cpp_alloc(size_t size) { void* p = DebugAllocate(size, MallocBlock::kMallocType); if (p != NULL) { return p; } struct debug_alloc_retry_data data; data.size = size; data.new_type = MallocBlock::kMallocType; return handle_oom(retry_debug_allocate, &data, false, true); } // Exported routines // frame forcer and force_frame exist only to prevent tail calls to // DebugDeallocate to be actually implemented as tail calls. This is // important because stack trace capturing in MallocBlockQueueEntry // relies on google_malloc section being on stack and tc_XXX functions // are in that section. So they must not jump to DebugDeallocate but // have to do call. frame_forcer call at the end of such functions // prevents tail calls to DebugDeallocate. static int frame_forcer; static void force_frame() { int dummy = *(int volatile *)&frame_forcer; (void)dummy; } extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) PERFTOOLS_NOTHROW { if (ThreadCache::IsUseEmergencyMalloc()) { return tcmalloc::EmergencyMalloc(size); } void* ptr = do_debug_malloc_or_debug_cpp_alloc(size); MallocHook::InvokeNewHook(ptr, size); return ptr; } extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) PERFTOOLS_NOTHROW { if (tcmalloc::IsEmergencyPtr(ptr)) { return tcmalloc::EmergencyFree(ptr); } MallocHook::InvokeDeleteHook(ptr); DebugDeallocate(ptr, MallocBlock::kMallocType, 0); force_frame(); } extern "C" PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) PERFTOOLS_NOTHROW { MallocHook::InvokeDeleteHook(ptr); DebugDeallocate(ptr, MallocBlock::kMallocType, size); force_frame(); } extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) PERFTOOLS_NOTHROW { if (ThreadCache::IsUseEmergencyMalloc()) { return tcmalloc::EmergencyCalloc(count, size); } // Overflow check const size_t total_size = count * size; if (size != 0 && total_size / size != count) return NULL; void* block = do_debug_malloc_or_debug_cpp_alloc(total_size); MallocHook::InvokeNewHook(block, total_size); if (block) memset(block, 0, total_size); return block; } extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_NOTHROW { if (tcmalloc::IsEmergencyPtr(ptr)) { return tcmalloc::EmergencyFree(ptr); } MallocHook::InvokeDeleteHook(ptr); DebugDeallocate(ptr, MallocBlock::kMallocType, 0); force_frame(); } extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) PERFTOOLS_NOTHROW { if (tcmalloc::IsEmergencyPtr(ptr)) { return tcmalloc::EmergencyRealloc(ptr, size); } if (ptr == NULL) { ptr = do_debug_malloc_or_debug_cpp_alloc(size); MallocHook::InvokeNewHook(ptr, size); return ptr; } if (size == 0) { MallocHook::InvokeDeleteHook(ptr); DebugDeallocate(ptr, MallocBlock::kMallocType, 0); return NULL; } MallocBlock* old = MallocBlock::FromRawPointer(ptr); old->Check(MallocBlock::kMallocType); MallocBlock* p = MallocBlock::Allocate(size, MallocBlock::kMallocType); // If realloc fails we are to leave the old block untouched and // return null if (p == NULL) return NULL; // if ptr was allocated via memalign, then old->data_size() is not // start of user data. So we must be careful to copy only user-data char *old_begin = (char *)old->data_addr(); char *old_end = old_begin + old->data_size(); ssize_t old_ssize = old_end - (char *)ptr; CHECK_CONDITION(old_ssize >= 0); size_t old_size = (size_t)old_ssize; CHECK_CONDITION(old_size <= old->data_size()); memcpy(p->data_addr(), ptr, (old_size < size) ? old_size : size); MallocHook::InvokeDeleteHook(ptr); MallocHook::InvokeNewHook(p->data_addr(), size); DebugDeallocate(ptr, MallocBlock::kMallocType, 0); MALLOC_TRACE("realloc", p->data_size(), p->data_addr()); return p->data_addr(); } extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) { void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, false); MallocHook::InvokeNewHook(ptr, size); if (ptr == NULL) { RAW_LOG(FATAL, "Unable to allocate %" PRIuS " bytes: new failed.", size); } return ptr; } extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW { void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, true); MallocHook::InvokeNewHook(ptr, size); return ptr; } extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_NOTHROW { MallocHook::InvokeDeleteHook(p); DebugDeallocate(p, MallocBlock::kNewType, 0); force_frame(); } extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void* p, size_t size) PERFTOOLS_NOTHROW { MallocHook::InvokeDeleteHook(p); DebugDeallocate(p, MallocBlock::kNewType, size); force_frame(); } // Some STL implementations explicitly invoke this. // It is completely equivalent to a normal delete (delete never throws). extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW { MallocHook::InvokeDeleteHook(p); DebugDeallocate(p, MallocBlock::kNewType, 0); force_frame(); } extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) { void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, false); MallocHook::InvokeNewHook(ptr, size); if (ptr == NULL) { RAW_LOG(FATAL, "Unable to allocate %" PRIuS " bytes: new[] failed.", size); } return ptr; } extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&) PERFTOOLS_NOTHROW { void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, true); MallocHook::InvokeNewHook(ptr, size); return ptr; } extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_NOTHROW { MallocHook::InvokeDeleteHook(p); DebugDeallocate(p, MallocBlock::kArrayNewType, 0); force_frame(); } extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void* p, size_t size) PERFTOOLS_NOTHROW { MallocHook::InvokeDeleteHook(p); DebugDeallocate(p, MallocBlock::kArrayNewType, size); force_frame(); } // Some STL implementations explicitly invoke this. // It is completely equivalent to a normal delete (delete never throws). extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) PERFTOOLS_NOTHROW { MallocHook::InvokeDeleteHook(p); DebugDeallocate(p, MallocBlock::kArrayNewType, 0); force_frame(); } // This is mostly the same as do_memalign in tcmalloc.cc. static void *do_debug_memalign(size_t alignment, size_t size, int type) { // Allocate >= size bytes aligned on "alignment" boundary // "alignment" is a power of two. void *p = 0; RAW_CHECK((alignment & (alignment-1)) == 0, "must be power of two"); const size_t data_offset = MallocBlock::data_offset(); // Allocate "alignment-1" extra bytes to ensure alignment is possible, and // a further data_offset bytes for an additional fake header. size_t extra_bytes = data_offset + alignment - 1; if (size + extra_bytes < size) return NULL; // Overflow p = DebugAllocate(size + extra_bytes, type); if (p != 0) { intptr_t orig_p = reinterpret_cast(p); // Leave data_offset bytes for fake header, and round up to meet // alignment. p = reinterpret_cast(RoundUp(orig_p + data_offset, alignment)); // Create a fake header block with an offset_ that points back to the // real header. FromRawPointer uses this value. MallocBlock *fake_hdr = reinterpret_cast( reinterpret_cast(p) - data_offset); // offset_ is distance between real and fake headers. // p is now end of fake header (beginning of client area), // and orig_p is the end of the real header, so offset_ // is their difference. // // Note that other fields of fake_hdr are initialized with // kMagicUninitializedByte fake_hdr->set_offset(reinterpret_cast(p) - orig_p); } return p; } struct memalign_retry_data { size_t align; size_t size; int type; }; static void *retry_debug_memalign(void *arg) { memalign_retry_data *data = static_cast(arg); return do_debug_memalign(data->align, data->size, data->type); } ATTRIBUTE_ALWAYS_INLINE inline void* do_debug_memalign_or_debug_cpp_memalign(size_t align, size_t size, int type, bool from_operator, bool nothrow) { void* p = do_debug_memalign(align, size, type); if (p != NULL) { return p; } struct memalign_retry_data data; data.align = align; data.size = size; data.type = type; return handle_oom(retry_debug_memalign, &data, from_operator, nothrow); } extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, size_t size) PERFTOOLS_NOTHROW { void *p = do_debug_memalign_or_debug_cpp_memalign(align, size, MallocBlock::kMallocType, false, true); MallocHook::InvokeNewHook(p, size); return p; } // Implementation taken from tcmalloc/tcmalloc.cc extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(void** result_ptr, size_t align, size_t size) PERFTOOLS_NOTHROW { if (((align % sizeof(void*)) != 0) || ((align & (align - 1)) != 0) || (align == 0)) { return EINVAL; } void* result = do_debug_memalign_or_debug_cpp_memalign(align, size, MallocBlock::kMallocType, false, true); MallocHook::InvokeNewHook(result, size); if (result == NULL) { return ENOMEM; } else { *result_ptr = result; return 0; } } extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) PERFTOOLS_NOTHROW { // Allocate >= size bytes starting on a page boundary void *p = do_debug_memalign_or_debug_cpp_memalign(getpagesize(), size, MallocBlock::kMallocType, false, true); MallocHook::InvokeNewHook(p, size); return p; } extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) PERFTOOLS_NOTHROW { // Round size up to a multiple of pages // then allocate memory on a page boundary int pagesize = getpagesize(); size = RoundUp(size, pagesize); if (size == 0) { // pvalloc(0) should allocate one page, according to size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html } void *p = do_debug_memalign_or_debug_cpp_memalign(pagesize, size, MallocBlock::kMallocType, false, true); MallocHook::InvokeNewHook(p, size); return p; } #if defined(ENABLE_ALIGNED_NEW_DELETE) extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned(size_t size, std::align_val_t align) { void* result = do_debug_memalign_or_debug_cpp_memalign(static_cast(align), size, MallocBlock::kNewType, true, false); MallocHook::InvokeNewHook(result, size); return result; } extern "C" PERFTOOLS_DLL_DECL void* tc_new_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t&) PERFTOOLS_NOTHROW { void* result = do_debug_memalign_or_debug_cpp_memalign(static_cast(align), size, MallocBlock::kNewType, true, true); MallocHook::InvokeNewHook(result, size); return result; } extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW { tc_delete(p); } extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW { // Reproduce actual size calculation done by do_debug_memalign const size_t alignment = static_cast(align); const size_t data_offset = MallocBlock::data_offset(); const size_t extra_bytes = data_offset + alignment - 1; tc_delete_sized(p, size + extra_bytes); } extern "C" PERFTOOLS_DLL_DECL void tc_delete_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW { tc_delete(p); } extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned(size_t size, std::align_val_t align) { void* result = do_debug_memalign_or_debug_cpp_memalign(static_cast(align), size, MallocBlock::kArrayNewType, true, false); MallocHook::InvokeNewHook(result, size); return result; } extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t align, const std::nothrow_t& nt) PERFTOOLS_NOTHROW { void* result = do_debug_memalign_or_debug_cpp_memalign(static_cast(align), size, MallocBlock::kArrayNewType, true, true); MallocHook::InvokeNewHook(result, size); return result; } extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned(void* p, std::align_val_t) PERFTOOLS_NOTHROW { tc_deletearray(p); } extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t align) PERFTOOLS_NOTHROW { // Reproduce actual size calculation done by do_debug_memalign const size_t alignment = static_cast(align); const size_t data_offset = MallocBlock::data_offset(); const size_t extra_bytes = data_offset + alignment - 1; tc_deletearray_sized(p, size + extra_bytes); } extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_aligned_nothrow(void* p, std::align_val_t, const std::nothrow_t&) PERFTOOLS_NOTHROW { tc_deletearray(p); } #endif // defined(ENABLE_ALIGNED_NEW_DELETE) // malloc_stats just falls through to the base implementation. extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_NOTHROW { do_malloc_stats(); } extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW { return do_mallopt(cmd, value); } #ifdef HAVE_STRUCT_MALLINFO extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW { return do_mallinfo(); } #endif extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_NOTHROW { return MallocExtension::instance()->GetAllocatedSize(ptr); } extern "C" PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_NOTHROW { void* result = DebugAllocate(size, MallocBlock::kMallocType); MallocHook::InvokeNewHook(result, size); return result; }