diff options
author | unknown <jonas@eel.(none)> | 2006-01-22 18:23:48 +0100 |
---|---|---|
committer | unknown <jonas@eel.(none)> | 2006-01-22 18:23:48 +0100 |
commit | fef9b7499c7c4008b71c4ff8f20889047a9af680 (patch) | |
tree | bfff75923baf31fd41abb5c2989524ca66e49459 | |
parent | b14d5d64f169f9c279849a244fc6666bb56f9d66 (diff) | |
download | mariadb-git-fef9b7499c7c4008b71c4ff8f20889047a9af680.tar.gz |
wl1497 - ndb - dynamic mem in ndbd
Impl. NdbdSuperPool (subclass of SuperPool) that uses Ndbd_mem_manager
Impl. micro benchmark
storage/ndb/src/kernel/vm/Makefile.am:
Add NdbdSuperPool , which impl. SuperPool
storage/ndb/src/kernel/vm/SuperPool.cpp:
remove virtuallity
storage/ndb/src/kernel/vm/SuperPool.hpp:
Remove unneccesary virtuality
storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp:
Add util methods used by NdbdSuperPool
storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp:
Add util methods used by NdbdSuperPool
storage/ndb/src/kernel/vm/NdbdSuperPool.cpp:
New BitKeeper file ``storage/ndb/src/kernel/vm/NdbdSuperPool.cpp''
storage/ndb/src/kernel/vm/NdbdSuperPool.hpp:
New BitKeeper file ``storage/ndb/src/kernel/vm/NdbdSuperPool.hpp''
storage/ndb/src/kernel/vm/bench_pool.cpp:
New BitKeeper file ``storage/ndb/src/kernel/vm/bench_pool.cpp''
-rw-r--r-- | storage/ndb/src/kernel/vm/Makefile.am | 13 | ||||
-rw-r--r-- | storage/ndb/src/kernel/vm/NdbdSuperPool.cpp | 228 | ||||
-rw-r--r-- | storage/ndb/src/kernel/vm/NdbdSuperPool.hpp | 55 | ||||
-rw-r--r-- | storage/ndb/src/kernel/vm/SuperPool.cpp | 4 | ||||
-rw-r--r-- | storage/ndb/src/kernel/vm/SuperPool.hpp | 3 | ||||
-rw-r--r-- | storage/ndb/src/kernel/vm/bench_pool.cpp | 249 | ||||
-rw-r--r-- | storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp | 21 | ||||
-rw-r--r-- | storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp | 9 |
8 files changed, 574 insertions, 8 deletions
diff --git a/storage/ndb/src/kernel/vm/Makefile.am b/storage/ndb/src/kernel/vm/Makefile.am index f2bef8ba1f7..3e89508e851 100644 --- a/storage/ndb/src/kernel/vm/Makefile.am +++ b/storage/ndb/src/kernel/vm/Makefile.am @@ -20,7 +20,8 @@ libkernel_a_SOURCES = \ Mutex.cpp SafeCounter.cpp \ Rope.cpp \ SuperPool.cpp \ - ndbd_malloc.cpp ndbd_malloc_impl.cpp + ndbd_malloc.cpp ndbd_malloc_impl.cpp \ + NdbdSuperPool.cpp INCLUDES_LOC = -I$(top_srcdir)/storage/ndb/src/mgmapi @@ -44,7 +45,7 @@ libkernel.dsp: Makefile \ @$(top_srcdir)/storage/ndb/config/win-sources $@ $(libkernel_a_SOURCES) @$(top_srcdir)/storage/ndb/config/win-libraries $@ LIB $(LDADD) -EXTRA_PROGRAMS = ndbd_malloc_impl_test +EXTRA_PROGRAMS = ndbd_malloc_impl_test bench_pool ndbd_malloc_impl_test_CXXFLAGS = -DUNIT_TEST ndbd_malloc_impl_test_SOURCES = ndbd_malloc_impl.cpp ndbd_malloc_impl_test_LDFLAGS = @ndb_bin_am_ldflags@ \ @@ -52,3 +53,11 @@ ndbd_malloc_impl_test_LDFLAGS = @ndb_bin_am_ldflags@ \ $(top_builddir)/mysys/libmysys.a \ $(top_builddir)/dbug/libdbug.a \ $(top_builddir)/strings/libmystrings.a + +bench_pool_SOURCES = bench_pool.cpp ndbd_malloc.cpp \ + SuperPool.cpp NdbdSuperPool.cpp ndbd_malloc_impl.cpp +bench_pool_LDFLAGS = @ndb_bin_am_ldflags@ \ + $(top_builddir)/storage/ndb/src/libndbclient.la \ + $(top_builddir)/mysys/libmysys.a \ + $(top_builddir)/dbug/libdbug.a \ + $(top_builddir)/strings/libmystrings.a diff --git a/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp b/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp new file mode 100644 index 00000000000..29174ab2e43 --- /dev/null +++ b/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp @@ -0,0 +1,228 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include <ndb_global.h> +#include "SuperPool.hpp" +#include "ndbd_malloc_impl.hpp" +#include "NdbdSuperPool.hpp" + +#define PSI (1 << (BMW_2LOG + 2)) + +struct AllocArea +{ + AllocArea(AllocArea* next); + + Uint16 m_currPage; // 2 + Uint16 m_numPages; // number of pages // 2 + SuperPool::PtrI m_firstPageI; // 4 + void* m_memory; // page-aligned pages // 4/8 + struct AllocArea* m_nextArea; // 4/8 + // tot 16/24 +}; + +AllocArea::AllocArea(AllocArea* next) +{ + m_nextArea = next; + m_firstPageI = RNIL; + m_currPage = m_numPages = 0; + m_memory = 0; +} + +NdbdSuperPool::NdbdSuperPool(class Ndbd_mem_manager & mm, + Uint32 pageSize, Uint32 pageBits) : + SuperPool(pageSize, pageBits), + m_mm(mm), + m_currArea(0), m_firstArea(0) +{ + m_memRoot = m_mm.get_memroot(); + + m_shift = Ndbd_mem_manager::log2((1 << (BMW_2LOG + 2)) / pageSize) - 1; + m_add = (1 << m_shift) - 1; +} + +NdbdSuperPool::~NdbdSuperPool() +{ + Uint32 cnt = PSI / sizeof(AllocArea); + AllocArea* ap = m_firstArea; + while(ap != 0) + { + AllocArea * first = ap; + for(Uint32 i = 0; i<cnt; i++) + { + if (ap->m_numPages) + { + m_mm.release(ap->m_memory, ap->m_numPages >> m_shift); + } + ap = ap->m_nextArea; + } + m_mm.release((void*)first, 1); + } +} + +bool +NdbdSuperPool::init_1() +{ + Uint32 pageCount = (1 << m_pageBits); + if (m_pageEnt == 0) { + // allocate page entry array + Uint32 bytes = pageCount * sizeof(PageEnt); + m_pageEnt = static_cast<PageEnt*>(malloc(bytes)); + if (m_pageEnt == 0) + return false; + for (Uint32 i = 0; i < pageCount; i++) + new (&m_pageEnt[i]) PageEnt(); + } + if (m_pageType == 0) { + // allocate type check array + Uint32 bytes = pageCount; + m_pageType = static_cast<Uint8*>(malloc(bytes)); + if (m_pageType == 0) + return false; + memset(m_pageType, 0, bytes); + } + + return true; +} + +static +void +initAllocAreaPage(AllocArea * p1) +{ + AllocArea * ap = p1; + Uint32 cnt = PSI / sizeof(AllocArea); + for(Uint32 i = 0; i<cnt; i++, ap++) + { + new (ap) AllocArea(ap + 1); + } + + (p1 + cnt - 1)->m_nextArea = 0; +} + +bool +NdbdSuperPool::init_2() +{ + m_memRoot = m_mm.get_memroot(); + + Uint32 cnt = 1; + AllocArea* p1 = (AllocArea*)m_mm.alloc(&cnt, 1); + if (p1 == 0) + return false; + + initAllocAreaPage(p1); + m_currArea = p1; + m_firstArea = p1; + return true; +} + +SuperPool::PtrI +NdbdSuperPool::getNewPage() +{ + AllocArea* ap = m_currArea; + Uint32 curr = ap->m_currPage; + Uint32 cnt = ap->m_numPages; + if (curr == cnt) + { + // area is used up + if (! (ap = allocMem())) + { + abort(); + return RNIL; + } + curr = ap->m_currPage; + cnt = ap->m_numPages; + } + + assert(curr < cnt); + PtrI pageI = ap->m_firstPageI; + Uint32 recBits = m_recBits; + Int32 ip = ((Int32)pageI >> recBits) + curr; + pageI = ip << recBits; + ap->m_currPage = curr + 1; + return pageI; +} + +Uint32 +NdbdSuperPool::allocAreaMemory(AllocArea* ap, Uint32 tryPages) +{ + Uint32 cnt = (tryPages + m_add) >> m_shift; + void* p1 = m_mm.alloc(&cnt, 1); + if (p1 == 0) + { + abort(); + return 0; + } + Uint32 pageI = getPageI(p1); + ap->m_firstPageI = pageI; + ap->m_currPage = 0; + ap->m_memory = p1; + ap->m_numPages = cnt << m_shift; + return cnt; +} + +AllocArea* +NdbdSuperPool::allocArea() +{ + AllocArea * curr = m_currArea; + AllocArea * next = curr->m_nextArea; + if (next == 0) + { + Uint32 cnt = 1; + AllocArea* p1 = (AllocArea*)m_mm.alloc(&cnt, 1); + if (p1 == 0) + return 0; + + initAllocAreaPage(p1); + + m_currArea->m_nextArea = p1; + return m_currArea = p1; + } + else + { + m_currArea = m_currArea->m_nextArea; + return m_currArea; + } +} + +AllocArea* +NdbdSuperPool::allocMem() +{ + // compute number of additional pages needed + if (m_totPages >= m_maxPages) + { + abort(); + return 0; + } + Uint32 needPages = (m_totPages == 0 ? m_initPages : m_incrPages); + + // add new area + AllocArea* ap = allocArea(); + if (ap == 0) + { + abort(); + return 0; + } + + Uint32 numPages; + if (!(numPages = allocAreaMemory(ap, needPages))) + { + abort(); + return 0; + } + + // update counts + m_totPages += numPages; + return ap; +} diff --git a/storage/ndb/src/kernel/vm/NdbdSuperPool.hpp b/storage/ndb/src/kernel/vm/NdbdSuperPool.hpp new file mode 100644 index 00000000000..fc552357a4f --- /dev/null +++ b/storage/ndb/src/kernel/vm/NdbdSuperPool.hpp @@ -0,0 +1,55 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#ifndef NDBD_SUPER_POOL_HPP +#define NDBD_SUPER_POOL_HPP + +#include "SuperPool.hpp" + +struct AllocArea; + +class NdbdSuperPool : public SuperPool +{ +public: + NdbdSuperPool(class Ndbd_mem_manager&, Uint32 pageSize, Uint32 pageBits); + + // Destructor. + virtual ~NdbdSuperPool(); + + // Get new page from current area. + virtual PtrI getNewPage(); + + // Call first...on all superpools (uses malloc) + bool init_1(); + + // Call second...uses mm + bool init_2(); + + virtual bool allocMemory() { return allocMem() != 0; } +private: + Uint32 allocAreaMemory(AllocArea*, Uint32 pages); + AllocArea* allocArea(); + AllocArea* allocMem(); + + // List of malloc areas. + Uint32 m_shift, m_add; + class Ndbd_mem_manager & m_mm; + + AllocArea* m_currArea; + AllocArea* m_firstArea; +}; + +#endif diff --git a/storage/ndb/src/kernel/vm/SuperPool.cpp b/storage/ndb/src/kernel/vm/SuperPool.cpp index c61bfcc3541..00b6fbc780e 100644 --- a/storage/ndb/src/kernel/vm/SuperPool.cpp +++ b/storage/ndb/src/kernel/vm/SuperPool.cpp @@ -644,7 +644,7 @@ HeapPool::getNewPage() if (ap->m_currPage == ap->m_numPages) { // area is used up if (ap->m_nextArea == 0) { - if (! allocMemory()) + if (! allocMemoryImpl()) return RNIL; } ap = m_currArea = ap->m_nextArea; @@ -711,7 +711,7 @@ HeapPool::allocArea(Area* ap, Uint32 tryPages) } bool -HeapPool::allocMemory() +HeapPool::allocMemoryImpl() { if (! allocInit()) return false; diff --git a/storage/ndb/src/kernel/vm/SuperPool.hpp b/storage/ndb/src/kernel/vm/SuperPool.hpp index 187383f5f71..8f649a2b32a 100644 --- a/storage/ndb/src/kernel/vm/SuperPool.hpp +++ b/storage/ndb/src/kernel/vm/SuperPool.hpp @@ -580,7 +580,8 @@ public: bool allocArea(Area* ap, Uint32 tryPages); // Allocate memory. - virtual bool allocMemory(); + virtual bool allocMemory() { return allocMemoryImpl();} + bool allocMemoryImpl(); // List of malloc areas. Area m_areaHead; diff --git a/storage/ndb/src/kernel/vm/bench_pool.cpp b/storage/ndb/src/kernel/vm/bench_pool.cpp new file mode 100644 index 00000000000..1042059b0c5 --- /dev/null +++ b/storage/ndb/src/kernel/vm/bench_pool.cpp @@ -0,0 +1,249 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + + +#include "NdbdSuperPool.hpp" +#include "ArrayPool.hpp" +#include <NdbTick.h> +#include "ndbd_malloc_impl.hpp" + +template <typename T> +inline +void +init(ArrayPool<T> & pool, Uint32 cnt) +{ + pool.setSize(cnt + 1); +} + +template <typename T> +inline +void +init(RecordPool<T> & pool, Uint32 cnt) +{ +} + + +template<typename T, typename R> +inline +void +test_pool(R& pool, Uint32 cnt, Uint32 loops) +{ + init(pool, cnt); + Ptr<T> ptr; + Uint32 *arr = (Uint32*)alloca(cnt * sizeof(Uint32)); + { + printf(" ; seize "); fflush(stdout); + Uint64 sum = 0; + for(Uint32 i = 0; i<loops; i++) + { + Uint64 start = NdbTick_CurrentMillisecond(); + for(Uint32 j = 0; j<cnt; j++) + { + bool b = pool.seize(ptr); + arr[j] = ptr.i; + assert(b); + } + Uint64 stop = NdbTick_CurrentMillisecond(); + + for(Uint32 j = 0; j<cnt; j++) + { + ptr.i = arr[j]; + pool.getPtr(ptr); + pool.release(ptr); + arr[j] = RNIL; + } + + sum += (stop - start); + if (i == 0) + printf("; first ; %lld", (stop - start)); + } + printf(" ; avg ; %lld ; tot ; %lld", sum/loops, sum);fflush(stdout); + } + + { + printf(" ; release "); fflush(stdout); + Uint64 sum = 0; + for(Uint32 i = 0; i<loops; i++) + { + for(Uint32 j = 0; j<cnt; j++) + { + bool b = pool.seize(ptr); + arr[j] = ptr.i; + assert(b); + } + + Uint64 start = NdbTick_CurrentMillisecond(); + for(Uint32 j = 0; j<cnt; j++) + { + ptr.i = arr[j]; + pool.release(ptr); + arr[j] = RNIL; + } + Uint64 stop = NdbTick_CurrentMillisecond(); + + sum += (stop - start); + } + printf("; avg ; %lld ; tot ; %lld", sum/loops, sum); fflush(stdout); + } + + { + printf(" ; mix"); fflush(stdout); + + Uint64 sum = 0; + Uint64 start = NdbTick_CurrentMillisecond(); + for(Uint32 i = 0; i<loops * cnt; i++) + { + int pos = rand() % cnt; + ptr.i = arr[pos]; + if (ptr.i == RNIL) + { + pool.seize(ptr); + arr[pos] = ptr.i; + assert(ptr.i != RNIL); + } + else + { + pool.release(ptr); + arr[pos] = RNIL; + } + } + Uint64 stop = NdbTick_CurrentMillisecond(); + + for(Uint32 j = 0; j<cnt; j++) + { + ptr.i = arr[j]; + if (ptr.i != RNIL) + { + pool.getPtr(ptr); + pool.release(ptr); + } + arr[j] = RNIL; + } + + sum += (stop - start); + printf(" ; %lld", sum); fflush(stdout); + } + + { + printf(" ; getPtr"); fflush(stdout); + + for(Uint32 j = 0; j<cnt; j++) + { + bool b = pool.seize(ptr); + arr[j] = ptr.i; + assert(b); + } + + Uint64 sum = 0; + Uint64 start = NdbTick_CurrentMillisecond(); + for(Uint32 i = 0; i<loops * cnt; i++) + { + int pos = rand() % cnt; + ptr.i = arr[pos]; + pool.getPtr(ptr); + } + Uint64 stop = NdbTick_CurrentMillisecond(); + + for(Uint32 j = 0; j<cnt; j++) + { + ptr.i = arr[j]; + pool.getPtr(ptr); + pool.release(ptr); + arr[j] = RNIL; + } + + sum += (stop - start); + printf(" ; %lld", sum); fflush(stdout); + } + ndbout_c(""); +} + +template <Uint32 sz> struct Rec { char data[sz-4]; Uint32 nextPool; }; +typedef Rec<32> Rec32; +typedef Rec<36> Rec36; +typedef Rec<256> Rec256; +typedef Rec<260> Rec260; + +Ndbd_mem_manager mem; + +template <typename T> +inline +void test_rp(Uint32 cnt, Uint32 loop, Uint32 pgsz) +{ + printf("RP ; %d ; ws ; %d ; page ; %d", + sizeof(T), (sizeof(T)*cnt) >> 10, pgsz >> 10); + NdbdSuperPool sp(mem, pgsz, 19); + GroupPool gp(sp); + sp.init_1(); + sp.init_2(); + + sp.setInitPages(4); + sp.setIncrPages(4); + sp.setMaxPages(~0); + sp.allocMemory(); + + RecordPool<T> pool(gp); + test_pool<T, RecordPool<T> >(pool, cnt, loop); +} + +template <typename T> +inline +void test_ap(Uint32 cnt, Uint32 loop) +{ + printf("AP ; %d ; ws ; %d ; page ; n/a", sizeof(T), (cnt * sizeof(T))>>10); + ArrayPool<T> pool; + test_pool<T, ArrayPool<T> >(pool, cnt, loop); +} + +int +main(int argc, char **argv) +{ + mem.init(10000); + + Uint32 cnt = 100; + Uint32 loop = 300000; + + while(cnt <= 1000000) + { + test_rp<Rec32>(cnt, loop, 8192); + test_rp<Rec32>(cnt, loop, 32768); + test_ap<Rec32>(cnt, loop); + + test_rp<Rec36>(cnt, loop, 8192); + test_rp<Rec36>(cnt, loop, 32768); + test_ap<Rec36>(cnt, loop); + + test_rp<Rec256>(cnt, loop, 8192); + test_rp<Rec256>(cnt, loop, 32768); + test_ap<Rec256>(cnt, loop); + + test_rp<Rec260>(cnt, loop, 8192); + test_rp<Rec260>(cnt, loop, 32768); + test_ap<Rec260>(cnt, loop); + + cnt *= 100; + loop /= 100; + } +} + +void +ErrorReporter::handleAssert(const char * msg, const char * file, + int line, int) +{ + ndbout << "ErrorReporter::handleAssert activated - " + << " line= " << line << endl; + abort(); +} diff --git a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp index 5443fdf1300..e78a1cc1194 100644 --- a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp +++ b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp @@ -84,7 +84,7 @@ Ndbd_mem_manager::init(Uint32 pages) release(start+1, end - 1 - start); } - return 0; + return true; } void @@ -200,6 +200,25 @@ Ndbd_mem_manager::alloc(Uint32* ret, Uint32 *pages, Uint32 min) * pages = 0; } +void* +Ndbd_mem_manager::alloc(Uint32 *pages, Uint32 min) +{ + Uint32 ret; + alloc(&ret, pages, min); + if (pages) + { + return m_base_page + ret; + } + return 0; +} + +void +Ndbd_mem_manager::release(void* ptr, Uint32 cnt) +{ + Uint32 page = ((Alloc_page*)ptr) - m_base_page; + release(page, cnt); +} + void Ndbd_mem_manager::insert_free_list(Uint32 start, Uint32 size) { diff --git a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp index e497f62b92a..15b5c9c390e 100644 --- a/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp +++ b/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp @@ -64,8 +64,12 @@ public: bool grow(Uint32 pages = 0); void dump() const ; -private: - + + void* get_memroot() const { return (void*)m_base_page;} + + void* alloc(Uint32 * pages, Uint32 min_requested); + void release(void* ptr, Uint32 cnt); + /** * Compute 2log of size * @note size = 0 -> 0 @@ -73,6 +77,7 @@ private: */ static Uint32 log2(Uint32 size); +private: /** * Return pointer to free page data on page */ |