Imported WebKit commit fce473cb4d55aa9fe9d0b0322a2fffecb731b961 (http://svn.webkit.org/repository/webkit/trunk@106560)

24 files changed, 1139 insertions, 441 deletions

diff --git a/Source/JavaScriptCore/heap/AllocationSpace.cpp b/Source/JavaScriptCore/heap/AllocationSpace.cpp
deleted file mode 100644
index e363de274..000000000
--- a/Source/JavaScriptCore/heap/AllocationSpace.cpp
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Copyright (C) 2011 Apple Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include "AllocationSpace.h"
-
-#include "Heap.h"
-
-#define COLLECT_ON_EVERY_ALLOCATION 0
-
-namespace JSC {
-
-inline void* AllocationSpace::tryAllocate(MarkedSpace::SizeClass& sizeClass)
-{
-    m_heap->m_operationInProgress = Allocation;
-    void* result = m_markedSpace.allocate(sizeClass);
-    m_heap->m_operationInProgress = NoOperation;
-    return result;
-}
-
-void* AllocationSpace::allocateSlowCase(MarkedSpace::SizeClass& sizeClass)
-{
-#if COLLECT_ON_EVERY_ALLOCATION
-    m_heap->collectAllGarbage();
-    ASSERT(m_heap->m_operationInProgress == NoOperation);
-#endif
-    
-    void* result = tryAllocate(sizeClass);
-    
-    if (LIKELY(result != 0))
-        return result;
-    
-    AllocationEffort allocationEffort;
-    
-    if ((
-#if ENABLE(GGC)
-         m_markedSpace.nurseryWaterMark() < m_heap->m_minBytesPerCycle
-#else
-         m_markedSpace.waterMark() < m_markedSpace.highWaterMark()
-#endif
-         ) || !m_heap->m_isSafeToCollect)
-        allocationEffort = AllocationMustSucceed;
-    else
-        allocationEffort = AllocationCanFail;
-    
-    MarkedBlock* block = allocateBlock(sizeClass.cellSize, allocationEffort);
-    if (block) {
-        m_markedSpace.addBlock(sizeClass, block);
-        void* result = tryAllocate(sizeClass);
-        ASSERT(result);
-        return result;
-    }
-    
-    m_heap->collect(Heap::DoNotSweep);
-    
-    result = tryAllocate(sizeClass);
-    
-    if (result)
-        return result;
-    
-    ASSERT(m_markedSpace.waterMark() < m_markedSpace.highWaterMark());
-    
-    m_markedSpace.addBlock(sizeClass, allocateBlock(sizeClass.cellSize, AllocationMustSucceed));
-    
-    result = tryAllocate(sizeClass);
-    ASSERT(result);
-    return result;
-}
-
-MarkedBlock* AllocationSpace::allocateBlock(size_t cellSize, AllocationSpace::AllocationEffort allocationEffort)
-{
-    MarkedBlock* block;
-    
-    {
-        MutexLocker locker(m_heap->m_freeBlockLock);
-        if (m_heap->m_numberOfFreeBlocks) {
-            block = m_heap->m_freeBlocks.removeHead();
-            ASSERT(block);
-            m_heap->m_numberOfFreeBlocks--;
-        } else
-            block = 0;
-    }
-    if (block)
-        block = MarkedBlock::recycle(block, cellSize);
-    else if (allocationEffort == AllocationCanFail)
-        return 0;
-    else
-        block = MarkedBlock::create(m_heap, cellSize);
-    
-    m_blocks.add(block);
-    
-    return block;
-}
-
-void AllocationSpace::freeBlocks(MarkedBlock* head)
-{
-    MarkedBlock* next;
-    for (MarkedBlock* block = head; block; block = next) {
-        next = block->next();
-        
-        m_blocks.remove(block);
-        block->sweep();
-        MutexLocker locker(m_heap->m_freeBlockLock);
-        m_heap->m_freeBlocks.append(block);
-        m_heap->m_numberOfFreeBlocks++;
-    }
-}
-
-class TakeIfUnmarked {
-public:
-    typedef MarkedBlock* ReturnType;
-    
-    TakeIfUnmarked(MarkedSpace*);
-    void operator()(MarkedBlock*);
-    ReturnType returnValue();
-    
-private:
-    MarkedSpace* m_markedSpace;
-    DoublyLinkedList<MarkedBlock> m_empties;
-};
-
-inline TakeIfUnmarked::TakeIfUnmarked(MarkedSpace* newSpace)
-    : m_markedSpace(newSpace)
-{
-}
-
-inline void TakeIfUnmarked::operator()(MarkedBlock* block)
-{
-    if (!block->markCountIsZero())
-        return;
-    
-    m_markedSpace->removeBlock(block);
-    m_empties.append(block);
-}
-
-inline TakeIfUnmarked::ReturnType TakeIfUnmarked::returnValue()
-{
-    return m_empties.head();
-}
-
-void AllocationSpace::shrink()
-{
-    // We record a temporary list of empties to avoid modifying m_blocks while iterating it.
-    TakeIfUnmarked takeIfUnmarked(&m_markedSpace);
-    freeBlocks(forEachBlock(takeIfUnmarked));
-}
-
-#if ENABLE(GGC)
-class GatherDirtyCells {
-    WTF_MAKE_NONCOPYABLE(GatherDirtyCells);
-public:
-    typedef void* ReturnType;
-    
-    explicit GatherDirtyCells(MarkedBlock::DirtyCellVector*);
-    void operator()(MarkedBlock*);
-    ReturnType returnValue() { return 0; }
-    
-private:
-    MarkedBlock::DirtyCellVector* m_dirtyCells;
-};
-
-inline GatherDirtyCells::GatherDirtyCells(MarkedBlock::DirtyCellVector* dirtyCells)
-    : m_dirtyCells(dirtyCells)
-{
-}
-
-inline void GatherDirtyCells::operator()(MarkedBlock* block)
-{
-    block->gatherDirtyCells(*m_dirtyCells);
-}
-
-void AllocationSpace::gatherDirtyCells(MarkedBlock::DirtyCellVector& dirtyCells)
-{
-    GatherDirtyCells gatherDirtyCells(&dirtyCells);
-    forEachBlock(gatherDirtyCells);
-}
-#endif
-
-}
diff --git a/Source/JavaScriptCore/heap/AllocationSpace.h b/Source/JavaScriptCore/heap/AllocationSpace.h
deleted file mode 100644
index 550cb9aa3..000000000
--- a/Source/JavaScriptCore/heap/AllocationSpace.h
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright (C) 2011 Apple Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef AllocationSpace_h
-#define AllocationSpace_h
-
-#include "MarkedBlockSet.h"
-#include "MarkedSpace.h"
-
-#include <wtf/HashSet.h>
-
-namespace JSC {
-
-class Heap;
-class MarkedBlock;
-
-class AllocationSpace {
-public:
-    AllocationSpace(Heap* heap)
-        : m_heap(heap)
-        , m_markedSpace(heap)
-    {
-    }
-    
-    typedef HashSet<MarkedBlock*>::iterator BlockIterator;
-    
-    MarkedBlockSet& blocks() { return m_blocks; }
-    MarkedSpace::SizeClass& sizeClassFor(size_t bytes) { return m_markedSpace.sizeClassFor(bytes); }
-    void setHighWaterMark(size_t bytes) { m_markedSpace.setHighWaterMark(bytes); }
-    size_t highWaterMark() { return m_markedSpace.highWaterMark(); }
-
-#if ENABLE(GGC)
-    void gatherDirtyCells(MarkedBlock::DirtyCellVector&);
-#endif
-
-    template<typename Functor> typename Functor::ReturnType forEachCell(Functor&);
-    template<typename Functor> typename Functor::ReturnType forEachCell();
-    template<typename Functor> typename Functor::ReturnType forEachBlock(Functor&);
-    template<typename Functor> typename Functor::ReturnType forEachBlock();
-    
-    void canonicalizeCellLivenessData() { m_markedSpace.canonicalizeCellLivenessData(); }
-    void resetAllocator() { m_markedSpace.resetAllocator(); }
-    
-    void* allocate(size_t);
-    void freeBlocks(MarkedBlock*);
-    void shrink();
-    
-private:
-    enum AllocationEffort { AllocationMustSucceed, AllocationCanFail };
-    
-    void* allocate(MarkedSpace::SizeClass&);
-    void* tryAllocate(MarkedSpace::SizeClass&);
-    void* allocateSlowCase(MarkedSpace::SizeClass&);
-    MarkedBlock* allocateBlock(size_t cellSize, AllocationEffort);
-    
-    Heap* m_heap;
-    MarkedSpace m_markedSpace;
-    MarkedBlockSet m_blocks;
-};
-
-template<typename Functor> inline typename Functor::ReturnType AllocationSpace::forEachCell(Functor& functor)
-{
-    canonicalizeCellLivenessData();
-
-    BlockIterator end = m_blocks.set().end();
-    for (BlockIterator it = m_blocks.set().begin(); it != end; ++it)
-        (*it)->forEachCell(functor);
-    return functor.returnValue();
-}
-
-template<typename Functor> inline typename Functor::ReturnType AllocationSpace::forEachCell()
-{
-    Functor functor;
-    return forEachCell(functor);
-}
-
-template<typename Functor> inline typename Functor::ReturnType AllocationSpace::forEachBlock(Functor& functor)
-{
-    BlockIterator end = m_blocks.set().end();
-    for (BlockIterator it = m_blocks.set().begin(); it != end; ++it)
-        functor(*it);
-    return functor.returnValue();
-}
-
-template<typename Functor> inline typename Functor::ReturnType AllocationSpace::forEachBlock()
-{
-    Functor functor;
-    return forEachBlock(functor);
-}
-
-inline void* AllocationSpace::allocate(MarkedSpace::SizeClass& sizeClass)
-{
-    // This is a light-weight fast path to cover the most common case.
-    MarkedBlock::FreeCell* firstFreeCell = sizeClass.firstFreeCell;
-    if (UNLIKELY(!firstFreeCell))
-        return allocateSlowCase(sizeClass);
-    
-    sizeClass.firstFreeCell = firstFreeCell->next;
-    return firstFreeCell;
-}
-
-inline void* AllocationSpace::allocate(size_t bytes)
-{
-    MarkedSpace::SizeClass& sizeClass = sizeClassFor(bytes);
-    return allocate(sizeClass);
-}
-
-}
-
-#endif
diff --git a/Source/JavaScriptCore/heap/BumpBlock.h b/Source/JavaScriptCore/heap/BumpBlock.h
new file mode 100644
index 000000000..b9f271ca8
--- /dev/null
+++ b/Source/JavaScriptCore/heap/BumpBlock.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+#ifndef BumpBlock_h
+#define BumpBlock_h
+
+#include "HeapBlock.h"
+
+namespace JSC {
+
+class BumpSpace;
+
+class BumpBlock : public HeapBlock {
+    friend class BumpSpace;
+public:
+    BumpBlock(PageAllocationAligned& allocation)
+        : HeapBlock(allocation)
+        , m_offset(m_payload)
+        , m_isPinned(false)
+    {
+    }
+
+private:
+    void* m_offset;
+    uintptr_t m_isPinned;
+    char m_payload[1];
+};
+
+} // namespace JSC
+
+#endif
diff --git a/Source/JavaScriptCore/heap/BumpSpace.cpp b/Source/JavaScriptCore/heap/BumpSpace.cpp
new file mode 100644
index 000000000..4eb0284dd
--- /dev/null
+++ b/Source/JavaScriptCore/heap/BumpSpace.cpp
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+#include "config.h"
+#include "BumpSpace.h"
+
+#include "BumpSpaceInlineMethods.h"
+
+namespace JSC {
+
+CheckedBoolean BumpSpace::tryAllocateSlowCase(size_t bytes, void** outPtr)
+{
+    if (isOversize(bytes))
+        return tryAllocateOversize(bytes, outPtr);
+    
+    m_totalMemoryUtilized += static_cast<size_t>(static_cast<char*>(m_currentBlock->m_offset) - m_currentBlock->m_payload);
+    if (!addNewBlock()) {
+        *outPtr = 0;
+        return false;
+    }
+    m_toSpaceFilter.add(reinterpret_cast<Bits>(m_currentBlock));
+    m_toSpaceSet.add(m_currentBlock);
+    *outPtr = allocateFromBlock(m_currentBlock, bytes);
+    return true;
+}
+
+} // namespace JSC
diff --git a/Source/JavaScriptCore/heap/BumpSpace.h b/Source/JavaScriptCore/heap/BumpSpace.h
new file mode 100644
index 000000000..30e6b74fe
--- /dev/null
+++ b/Source/JavaScriptCore/heap/BumpSpace.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+#ifndef BumpSpace_h
+#define BumpSpace_h
+
+#include "HeapBlock.h"
+#include "TinyBloomFilter.h"
+#include <wtf/Assertions.h>
+#include <wtf/CheckedBoolean.h>
+#include <wtf/DoublyLinkedList.h>
+#include <wtf/HashSet.h>
+#include <wtf/OSAllocator.h>
+#include <wtf/PageAllocationAligned.h>
+#include <wtf/StdLibExtras.h>
+#include <wtf/ThreadingPrimitives.h>
+
+namespace JSC {
+
+class Heap;
+class BumpBlock;
+class HeapBlock;
+
+class BumpSpace {
+    friend class SlotVisitor;
+public:
+    BumpSpace(Heap*);
+    void init();
+
+    CheckedBoolean tryAllocate(size_t, void**);
+    CheckedBoolean tryReallocate(void**, size_t, size_t);
+    
+    void startedCopying();
+    void doneCopying();
+    bool isInCopyPhase() { return m_inCopyingPhase; }
+
+    void pin(BumpBlock*);
+    bool isPinned(void*);
+
+    bool contains(void*, BumpBlock*&);
+
+    size_t totalMemoryAllocated() { return m_totalMemoryAllocated; }
+    size_t totalMemoryUtilized() { return m_totalMemoryUtilized; }
+
+    static BumpBlock* blockFor(void*);
+
+private:
+    CheckedBoolean tryAllocateSlowCase(size_t, void**);
+    CheckedBoolean addNewBlock();
+    CheckedBoolean allocateNewBlock(BumpBlock**);
+    bool fitsInCurrentBlock(size_t);
+    
+    static void* allocateFromBlock(BumpBlock*, size_t);
+    CheckedBoolean tryAllocateOversize(size_t, void**);
+    CheckedBoolean tryReallocateOversize(void**, size_t, size_t);
+    
+    static bool isOversize(size_t);
+    
+    CheckedBoolean borrowBlock(BumpBlock**);
+    CheckedBoolean getFreshBlock(AllocationEffort, BumpBlock**);
+    void doneFillingBlock(BumpBlock*);
+    void recycleBlock(BumpBlock*);
+    static bool fitsInBlock(BumpBlock*, size_t);
+    static BumpBlock* oversizeBlockFor(void* ptr);
+
+    Heap* m_heap;
+
+    BumpBlock* m_currentBlock;
+
+    TinyBloomFilter m_toSpaceFilter;
+    TinyBloomFilter m_oversizeFilter;
+    HashSet<BumpBlock*> m_toSpaceSet;
+
+    Mutex m_toSpaceLock;
+    Mutex m_memoryStatsLock;
+
+    DoublyLinkedList<HeapBlock>* m_toSpace;
+    DoublyLinkedList<HeapBlock>* m_fromSpace;
+    
+    DoublyLinkedList<HeapBlock> m_blocks1;
+    DoublyLinkedList<HeapBlock> m_blocks2;
+    DoublyLinkedList<HeapBlock> m_oversizeBlocks;
+   
+    size_t m_totalMemoryAllocated;
+    size_t m_totalMemoryUtilized;
+
+    bool m_inCopyingPhase;
+
+    Mutex m_loanedBlocksLock; 
+    ThreadCondition m_loanedBlocksCondition;
+    size_t m_numberOfLoanedBlocks;
+
+    static const size_t s_blockSize = 64 * KB;
+    static const size_t s_maxAllocationSize = 32 * KB;
+    static const size_t s_pageSize = 4 * KB;
+    static const size_t s_pageMask = ~(s_pageSize - 1);
+    static const size_t s_initialBlockNum = 16;
+    static const size_t s_blockMask = ~(s_blockSize - 1);
+};
+
+} // namespace JSC
+
+#endif
diff --git a/Source/JavaScriptCore/heap/BumpSpaceInlineMethods.h b/Source/JavaScriptCore/heap/BumpSpaceInlineMethods.h
new file mode 100644
index 000000000..3454631b0
--- /dev/null
+++ b/Source/JavaScriptCore/heap/BumpSpaceInlineMethods.h
@@ -0,0 +1,400 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+#ifndef BumpSpaceInlineMethods_h
+#define BumpSpaceInlineMethods_h
+
+#include "BumpBlock.h"
+#include "BumpSpace.h"
+#include "Heap.h"
+#include "HeapBlock.h"
+#include "JSGlobalData.h"
+#include <wtf/CheckedBoolean.h>
+
+namespace JSC {
+
+inline BumpSpace::BumpSpace(Heap* heap)
+    : m_heap(heap)
+    , m_currentBlock(0)
+    , m_toSpace(0)
+    , m_fromSpace(0)
+    , m_totalMemoryAllocated(0)
+    , m_totalMemoryUtilized(0)
+    , m_inCopyingPhase(false)
+    , m_numberOfLoanedBlocks(0)
+{
+}
+
+inline void BumpSpace::init()
+{
+    m_toSpace = &m_blocks1;
+    m_fromSpace = &m_blocks2;
+    
+    m_totalMemoryAllocated += s_blockSize * s_initialBlockNum;
+
+    if (!addNewBlock())
+        CRASH();
+}   
+
+inline bool BumpSpace::contains(void* ptr, BumpBlock*& result)
+{
+    BumpBlock* block = blockFor(ptr);
+    result = block;
+    return !m_toSpaceFilter.ruleOut(reinterpret_cast<Bits>(block)) && m_toSpaceSet.contains(block);
+}
+
+inline void BumpSpace::pin(BumpBlock* block)
+{
+    block->m_isPinned = true;
+}
+
+inline void BumpSpace::startedCopying()
+{
+    DoublyLinkedList<HeapBlock>* temp = m_fromSpace;
+    m_fromSpace = m_toSpace;
+    m_toSpace = temp;
+
+    m_toSpaceFilter.reset();
+
+    m_totalMemoryUtilized = 0;
+
+    ASSERT(!m_inCopyingPhase);
+    ASSERT(!m_numberOfLoanedBlocks);
+    m_inCopyingPhase = true;
+}
+
+inline void BumpSpace::doneCopying()
+{
+    {
+        MutexLocker locker(m_loanedBlocksLock);
+        while (m_numberOfLoanedBlocks > 0)
+            m_loanedBlocksCondition.wait(m_loanedBlocksLock);
+    }
+
+    ASSERT(m_inCopyingPhase);
+    m_inCopyingPhase = false;
+    while (!m_fromSpace->isEmpty()) {
+        BumpBlock* block = static_cast<BumpBlock*>(m_fromSpace->removeHead());
+        if (block->m_isPinned) {
+            block->m_isPinned = false;
+            m_toSpace->push(block);
+            continue;
+        }
+
+        m_toSpaceSet.remove(block);
+        {
+            MutexLocker locker(m_heap->m_freeBlockLock);
+            m_heap->m_freeBlocks.push(block);
+            m_heap->m_numberOfFreeBlocks++;
+        }
+    }
+
+    BumpBlock* curr = static_cast<BumpBlock*>(m_oversizeBlocks.head());
+    while (curr) {
+        BumpBlock* next = static_cast<BumpBlock*>(curr->next());
+        if (!curr->m_isPinned) {
+            m_oversizeBlocks.remove(curr);
+            m_totalMemoryAllocated -= curr->m_allocation.size();
+            m_totalMemoryUtilized -= curr->m_allocation.size() - sizeof(BumpBlock);
+            curr->m_allocation.deallocate();
+        } else
+            curr->m_isPinned = false;
+        curr = next;
+    }
+
+    if (!(m_currentBlock = static_cast<BumpBlock*>(m_toSpace->head())))
+        if (!addNewBlock())
+            CRASH();
+}
+
+inline void BumpSpace::doneFillingBlock(BumpBlock* block)
+{
+    ASSERT(block);
+    ASSERT(block->m_offset < reinterpret_cast<char*>(block) + s_blockSize);
+    ASSERT(m_inCopyingPhase);
+
+    if (block->m_offset == block->m_payload) {
+        recycleBlock(block);
+        return;
+    }
+
+    {
+        MutexLocker locker(m_toSpaceLock);
+        m_toSpace->push(block);
+        m_toSpaceSet.add(block);
+        m_toSpaceFilter.add(reinterpret_cast<Bits>(block));
+    }
+
+    {
+        MutexLocker locker(m_memoryStatsLock);
+        m_totalMemoryUtilized += static_cast<size_t>(static_cast<char*>(block->m_offset) - block->m_payload);
+    }
+
+    {
+        MutexLocker locker(m_loanedBlocksLock);
+        ASSERT(m_numberOfLoanedBlocks > 0);
+        m_numberOfLoanedBlocks--;
+        if (!m_numberOfLoanedBlocks)
+            m_loanedBlocksCondition.signal();
+    }
+}
+
+inline void BumpSpace::recycleBlock(BumpBlock* block)
+{
+    {
+        MutexLocker locker(m_heap->m_freeBlockLock);
+        m_heap->m_freeBlocks.push(block);
+        m_heap->m_numberOfFreeBlocks++;
+    }
+
+    {
+        MutexLocker locker(m_loanedBlocksLock);
+        ASSERT(m_numberOfLoanedBlocks > 0);
+        m_numberOfLoanedBlocks--;
+        if (!m_numberOfLoanedBlocks)
+            m_loanedBlocksCondition.signal();
+    }
+}
+
+inline CheckedBoolean BumpSpace::getFreshBlock(AllocationEffort allocationEffort, BumpBlock** outBlock)
+{
+    HeapBlock* heapBlock = 0;
+    BumpBlock* block = 0;
+    {
+        MutexLocker locker(m_heap->m_freeBlockLock);
+        if (!m_heap->m_freeBlocks.isEmpty()) {
+            heapBlock = m_heap->m_freeBlocks.removeHead();
+            m_heap->m_numberOfFreeBlocks--;
+        }
+    }
+    if (heapBlock)
+        block = new (NotNull, heapBlock) BumpBlock(heapBlock->m_allocation);
+    else if (allocationEffort == AllocationMustSucceed) {
+        if (!allocateNewBlock(&block)) {
+            *outBlock = 0;
+            ASSERT_NOT_REACHED();
+            return false;
+        }
+    } else {
+        ASSERT(allocationEffort == AllocationCanFail);
+        if (m_heap->waterMark() >= m_heap->highWaterMark() && m_heap->m_isSafeToCollect)
+            m_heap->collect(Heap::DoNotSweep);
+        
+        if (!getFreshBlock(AllocationMustSucceed, &block)) {
+            *outBlock = 0;
+            ASSERT_NOT_REACHED();
+            return false;
+        }
+    }
+    ASSERT(block);
+    ASSERT(isPointerAligned(block->m_offset));
+    *outBlock = block;
+    return true;
+}
+
+inline CheckedBoolean BumpSpace::borrowBlock(BumpBlock** outBlock)
+{
+    BumpBlock* block = 0;
+    if (!getFreshBlock(AllocationMustSucceed, &block)) {
+        *outBlock = 0;
+        return false;
+    }
+
+    ASSERT(m_inCopyingPhase);
+    MutexLocker locker(m_loanedBlocksLock);
+    m_numberOfLoanedBlocks++;
+
+    ASSERT(block->m_offset == block->m_payload);
+    *outBlock = block;
+    return true;
+}
+
+inline CheckedBoolean BumpSpace::addNewBlock()
+{
+    BumpBlock* block = 0;
+    if (!getFreshBlock(AllocationCanFail, &block))
+        return false;
+        
+    m_toSpace->push(block);
+    m_currentBlock = block;
+    return true;
+}
+
+inline CheckedBoolean BumpSpace::allocateNewBlock(BumpBlock** outBlock)
+{
+    PageAllocationAligned allocation = PageAllocationAligned::allocate(s_blockSize, s_blockSize, OSAllocator::JSGCHeapPages);
+    if (!static_cast<bool>(allocation)) {
+        *outBlock = 0;
+        return false;
+    }
+
+    {
+        MutexLocker locker(m_memoryStatsLock);
+        m_totalMemoryAllocated += s_blockSize;
+    }
+
+    *outBlock = new (NotNull, allocation.base()) BumpBlock(allocation);
+    return true;
+}
+
+inline bool BumpSpace::fitsInBlock(BumpBlock* block, size_t bytes)
+{
+    return static_cast<char*>(block->m_offset) + bytes < reinterpret_cast<char*>(block) + s_blockSize && static_cast<char*>(block->m_offset) + bytes > block->m_offset;
+}
+
+inline bool BumpSpace::fitsInCurrentBlock(size_t bytes)
+{
+    return fitsInBlock(m_currentBlock, bytes);
+}
+
+inline CheckedBoolean BumpSpace::tryAllocate(size_t bytes, void** outPtr)
+{
+    ASSERT(!m_heap->globalData()->isInitializingObject());
+
+    if (isOversize(bytes) || !fitsInCurrentBlock(bytes))
+        return tryAllocateSlowCase(bytes, outPtr);
+    
+    *outPtr = allocateFromBlock(m_currentBlock, bytes);
+    return true;
+}
+
+inline CheckedBoolean BumpSpace::tryAllocateOversize(size_t bytes, void** outPtr)
+{
+    ASSERT(isOversize(bytes));
+    
+    size_t blockSize = WTF::roundUpToMultipleOf<s_pageSize>(sizeof(BumpBlock) + bytes);
+    PageAllocationAligned allocation = PageAllocationAligned::allocate(blockSize, s_pageSize, OSAllocator::JSGCHeapPages);
+    if (!static_cast<bool>(allocation)) {
+        *outPtr = 0;
+        return false;
+    }
+    BumpBlock* block = new (NotNull, allocation.base()) BumpBlock(allocation);
+    m_oversizeBlocks.push(block);
+    ASSERT(isPointerAligned(block->m_offset));
+
+    m_oversizeFilter.add(reinterpret_cast<Bits>(block));
+    
+    m_totalMemoryAllocated += blockSize;
+    m_totalMemoryUtilized += bytes;
+
+    *outPtr = block->m_offset;
+    return true;
+}
+
+inline void* BumpSpace::allocateFromBlock(BumpBlock* block, size_t bytes)
+{
+    ASSERT(!isOversize(bytes));
+    ASSERT(fitsInBlock(block, bytes));
+    ASSERT(isPointerAligned(block->m_offset));
+    
+    void* ptr = block->m_offset;
+    ASSERT(block->m_offset >= block->m_payload && block->m_offset < reinterpret_cast<char*>(block) + s_blockSize);
+    block->m_offset = static_cast<void*>((static_cast<char*>(ptr) + bytes));
+    ASSERT(block->m_offset >= block->m_payload && block->m_offset < reinterpret_cast<char*>(block) + s_blockSize);
+
+    ASSERT(isPointerAligned(ptr));
+    return ptr;
+}
+
+inline CheckedBoolean BumpSpace::tryReallocate(void** ptr, size_t oldSize, size_t newSize)
+{
+    if (oldSize >= newSize)
+        return true;
+    
+    void* oldPtr = *ptr;
+    ASSERT(!m_heap->globalData()->isInitializingObject());
+
+    if (isOversize(oldSize) || isOversize(newSize))
+        return tryReallocateOversize(ptr, oldSize, newSize);
+
+    if (static_cast<char*>(oldPtr) + oldSize == m_currentBlock->m_offset && oldPtr > m_currentBlock && oldPtr < reinterpret_cast<char*>(m_currentBlock) + s_blockSize) {
+        m_currentBlock->m_offset = oldPtr;
+        if (fitsInCurrentBlock(newSize)) {
+            m_totalMemoryUtilized += newSize - oldSize;
+            return allocateFromBlock(m_currentBlock, newSize);
+        }
+    }
+    m_totalMemoryUtilized -= oldSize;
+
+    void* result = 0;
+    if (!tryAllocate(newSize, &result)) {
+        *ptr = 0;
+        return false;
+    }
+    memcpy(result, oldPtr, oldSize);
+    *ptr = result;
+    return true;
+}
+
+inline CheckedBoolean BumpSpace::tryReallocateOversize(void** ptr, size_t oldSize, size_t newSize)
+{
+    ASSERT(isOversize(oldSize) || isOversize(newSize));
+    ASSERT(newSize > oldSize);
+
+    void* oldPtr = *ptr;
+    
+    void* newPtr = 0;
+    if (!tryAllocateOversize(newSize, &newPtr)) {
+        *ptr = 0;
+        return false;
+    }
+    memcpy(newPtr, oldPtr, oldSize);
+
+    if (isOversize(oldSize)) {
+        BumpBlock* oldBlock = oversizeBlockFor(oldPtr);
+        m_oversizeBlocks.remove(oldBlock);
+        oldBlock->m_allocation.deallocate();
+        m_totalMemoryAllocated -= oldSize + sizeof(BumpBlock);
+    }
+    
+    m_totalMemoryUtilized -= oldSize;
+
+    *ptr = newPtr;
+    return true;
+}
+
+inline bool BumpSpace::isOversize(size_t bytes)
+{
+    return bytes > s_maxAllocationSize;
+}
+
+inline bool BumpSpace::isPinned(void* ptr)
+{
+    return blockFor(ptr)->m_isPinned;
+}
+
+inline BumpBlock* BumpSpace::oversizeBlockFor(void* ptr)
+{
+    return reinterpret_cast<BumpBlock*>(reinterpret_cast<size_t>(ptr) & s_pageMask);
+}
+
+inline BumpBlock* BumpSpace::blockFor(void* ptr)
+{
+    return reinterpret_cast<BumpBlock*>(reinterpret_cast<size_t>(ptr) & s_blockMask);
+}
+
+} // namespace JSC
+
+#endif
diff --git a/Source/JavaScriptCore/heap/ConservativeRoots.cpp b/Source/JavaScriptCore/heap/ConservativeRoots.cpp
index 05c668c35..a509f06e1 100644
--- a/Source/JavaScriptCore/heap/ConservativeRoots.cpp
+++ b/Source/JavaScriptCore/heap/ConservativeRoots.cpp
@@ -26,6 +26,8 @@
 #include "config.h"
 #include "ConservativeRoots.h"
 
+#include "BumpSpace.h"
+#include "BumpSpaceInlineMethods.h"
 #include "CodeBlock.h"
 #include "DFGCodeBlocks.h"
 #include "JSCell.h"
@@ -34,16 +36,12 @@
 
 namespace JSC {
 
-inline bool isPointerAligned(void* p)
-{
-    return !((intptr_t)(p) & (sizeof(char*) - 1));
-}
-
-ConservativeRoots::ConservativeRoots(const MarkedBlockSet* blocks)
+ConservativeRoots::ConservativeRoots(const MarkedBlockSet* blocks, BumpSpace* bumpSpace)
     : m_roots(m_inlineRoots)
     , m_size(0)
     , m_capacity(inlineCapacity)
     , m_blocks(blocks)
+    , m_bumpSpace(bumpSpace)
 {
 }
 
@@ -74,6 +72,10 @@ inline void ConservativeRoots::genericAddPointer(void* p, TinyBloomFilter filter
 {
     markHook.mark(p);
     
+    BumpBlock* block;
+    if (m_bumpSpace->contains(p, block))
+        m_bumpSpace->pin(block);
+    
     MarkedBlock* candidate = MarkedBlock::blockFor(p);
     if (filter.ruleOut(reinterpret_cast<Bits>(candidate))) {
         ASSERT(!candidate || !m_blocks->set().contains(candidate));
@@ -110,8 +112,8 @@ void ConservativeRoots::genericAddSpan(void* begin, void* end, MarkHook& markHoo
 
 void ConservativeRoots::add(void* begin, void* end)
 {
-    DummyMarkHook dummyMarkHook;
-    genericAddSpan(begin, end, dummyMarkHook);
+    DummyMarkHook hook;
+    genericAddSpan(begin, end, hook);
 }
 
 void ConservativeRoots::add(void* begin, void* end, DFGCodeBlocks& dfgCodeBlocks)
diff --git a/Source/JavaScriptCore/heap/ConservativeRoots.h b/Source/JavaScriptCore/heap/ConservativeRoots.h
index 86dfc5886..40b0996d0 100644
--- a/Source/JavaScriptCore/heap/ConservativeRoots.h
+++ b/Source/JavaScriptCore/heap/ConservativeRoots.h
@@ -38,7 +38,7 @@ class Heap;
 
 class ConservativeRoots {
 public:
-    ConservativeRoots(const MarkedBlockSet*);
+    ConservativeRoots(const MarkedBlockSet*, BumpSpace*);
     ~ConservativeRoots();
 
     void add(void* begin, void* end);
@@ -63,6 +63,7 @@ private:
     size_t m_size;
     size_t m_capacity;
     const MarkedBlockSet* m_blocks;
+    BumpSpace* m_bumpSpace;
     JSCell* m_inlineRoots[inlineCapacity];
 };
 
diff --git a/Source/JavaScriptCore/heap/HandleHeap.h b/Source/JavaScriptCore/heap/HandleHeap.h
index c577791d8..3b9db37a2 100644
--- a/Source/JavaScriptCore/heap/HandleHeap.h
+++ b/Source/JavaScriptCore/heap/HandleHeap.h
@@ -40,7 +40,7 @@ class JSGlobalData;
 class JSValue;
 class SlotVisitor;
 
-class WeakHandleOwner {
+class JS_EXPORT_PRIVATE WeakHandleOwner {
 public:
     virtual ~WeakHandleOwner();
     virtual bool isReachableFromOpaqueRoots(Handle<Unknown>, void* context, SlotVisitor&);
@@ -65,7 +65,7 @@ public:
     void visitWeakHandles(HeapRootVisitor&);
     void finalizeWeakHandles();
 
-    void writeBarrier(HandleSlot, const JSValue&);
+    JS_EXPORT_PRIVATE void writeBarrier(HandleSlot, const JSValue&);
 
 #if !ASSERT_DISABLED
     bool hasWeakOwner(HandleSlot, WeakHandleOwner*);
@@ -111,7 +111,7 @@ private:
     static HandleSlot toHandle(Node*);
     static Node* toNode(HandleSlot);
 
-    void grow();
+    JS_EXPORT_PRIVATE void grow();
     
 #if ENABLE(GC_VALIDATION) || !ASSERT_DISABLED
     bool isValidWeakNode(Node*);
diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp
index 61eba08a4..a2136083a 100644
--- a/Source/JavaScriptCore/heap/Heap.cpp
+++ b/Source/JavaScriptCore/heap/Heap.cpp
@@ -21,6 +21,8 @@
 #include "config.h"
 #include "Heap.h"
 
+#include "BumpSpace.h"
+#include "BumpSpaceInlineMethods.h"
 #include "CodeBlock.h"
 #include "ConservativeRoots.h"
 #include "GCActivityCallback.h"
@@ -311,8 +313,11 @@ Heap::Heap(JSGlobalData* globalData, HeapSize heapSize)
     : m_heapSize(heapSize)
     , m_minBytesPerCycle(heapSizeForHint(heapSize))
     , m_lastFullGCSize(0)
+    , m_waterMark(0)
+    , m_highWaterMark(m_minBytesPerCycle)
     , m_operationInProgress(NoOperation)
     , m_objectSpace(this)
+    , m_storageSpace(this)
     , m_blockFreeingThreadShouldQuit(false)
     , m_extraCost(0)
     , m_markListSet(0)
@@ -324,12 +329,12 @@ Heap::Heap(JSGlobalData* globalData, HeapSize heapSize)
     , m_isSafeToCollect(false)
     , m_globalData(globalData)
 {
-    m_objectSpace.setHighWaterMark(m_minBytesPerCycle);
     (*m_activityCallback)();
     m_numberOfFreeBlocks = 0;
     m_blockFreeingThread = createThread(blockFreeingThreadStartFunc, this, "JavaScriptCore::BlockFree");
     
     ASSERT(m_blockFreeingThread);
+    m_storageSpace.init();
 }
 
 Heap::~Heap()
@@ -433,7 +438,7 @@ void Heap::blockFreeingThreadMain()
                 if (m_numberOfFreeBlocks <= desiredNumberOfFreeBlocks)
                     block = 0;
                 else {
-                    block = m_freeBlocks.removeHead();
+                    block = static_cast<MarkedBlock*>(m_freeBlocks.removeHead());
                     ASSERT(block);
                     m_numberOfFreeBlocks--;
                 }
@@ -460,7 +465,7 @@ void Heap::reportExtraMemoryCostSlowCase(size_t cost)
     // if a large value survives one garbage collection, there is not much point to
     // collecting more frequently as long as it stays alive.
 
-    if (m_extraCost > maxExtraCost && m_extraCost > m_objectSpace.highWaterMark() / 2)
+    if (m_extraCost > maxExtraCost && m_extraCost > highWaterMark() / 2)
         collectAllGarbage();
     m_extraCost += cost;
 }
@@ -509,7 +514,7 @@ void Heap::popTempSortVector(Vector<ValueStringPair>* tempVector)
     ASSERT_UNUSED(tempVector, tempVector == m_tempSortingVectors.last());
     m_tempSortingVectors.removeLast();
 }
-    
+
 void Heap::markTempSortVectors(HeapRootVisitor& heapRootVisitor)
 {
     typedef Vector<Vector<ValueStringPair>* > VectorOfValueStringVectors;
@@ -547,7 +552,7 @@ void Heap::getConservativeRegisterRoots(HashSet<JSCell*>& roots)
     if (m_operationInProgress != NoOperation)
         CRASH();
     m_operationInProgress = Collection;
-    ConservativeRoots registerFileRoots(&m_objectSpace.blocks());
+    ConservativeRoots registerFileRoots(&m_objectSpace.blocks(), &m_storageSpace);
     registerFile().gatherConservativeRoots(registerFileRoots);
     size_t registerFileRootCount = registerFileRoots.size();
     JSCell** registerRoots = registerFileRoots.roots();
@@ -573,13 +578,13 @@ void Heap::markRoots(bool fullGC)
     
     // We gather conservative roots before clearing mark bits because conservative
     // gathering uses the mark bits to determine whether a reference is valid.
-    ConservativeRoots machineThreadRoots(&m_objectSpace.blocks());
+    ConservativeRoots machineThreadRoots(&m_objectSpace.blocks(), &m_storageSpace);
     {
         GCPHASE(GatherConservativeRoots);
         m_machineThreads.gatherConservativeRoots(machineThreadRoots, &dummy);
     }
 
-    ConservativeRoots registerFileRoots(&m_objectSpace.blocks());
+    ConservativeRoots registerFileRoots(&m_objectSpace.blocks(), &m_storageSpace);
     m_dfgCodeBlocks.clearMarks();
     {
         GCPHASE(GatherRegisterFileRoots);
@@ -597,6 +602,7 @@ void Heap::markRoots(bool fullGC)
         clearMarks();
     }
 
+    m_storageSpace.startedCopying();
     SlotVisitor& visitor = m_slotVisitor;
     HeapRootVisitor heapRootVisitor(visitor);
 
@@ -700,8 +706,10 @@ void Heap::markRoots(bool fullGC)
     }
     GCCOUNTER(VisitedValueCount, visitor.visitCount());
 
+    visitor.doneCopying();
     visitor.reset();
     m_sharedData.reset();
+    m_storageSpace.doneCopying();
 
     m_operationInProgress = NoOperation;
 }
@@ -822,11 +830,11 @@ void Heap::collect(SweepToggle sweepToggle)
     // water mark to be proportional to the current size of the heap. The exact
     // proportion is a bit arbitrary. A 2X multiplier gives a 1:1 (heap size :
     // new bytes allocated) proportion, and seems to work well in benchmarks.
-    size_t newSize = size();
+    size_t newSize = size() + m_storageSpace.totalMemoryUtilized();
     size_t proportionalBytes = 2 * newSize;
     if (fullGC) {
         m_lastFullGCSize = newSize;
-        m_objectSpace.setHighWaterMark(max(proportionalBytes, m_minBytesPerCycle));
+        setHighWaterMark(max(proportionalBytes, m_minBytesPerCycle));
     }
     JAVASCRIPTCORE_GC_END();
 
@@ -887,7 +895,7 @@ void Heap::releaseFreeBlocks()
             if (!m_numberOfFreeBlocks)
                 block = 0;
             else {
-                block = m_freeBlocks.removeHead();
+                block = static_cast<MarkedBlock*>(m_freeBlocks.removeHead());
                 ASSERT(block);
                 m_numberOfFreeBlocks--;
             }
diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h
index 1b228253b..40a8376f0 100644
--- a/Source/JavaScriptCore/heap/Heap.h
+++ b/Source/JavaScriptCore/heap/Heap.h
@@ -22,7 +22,6 @@
 #ifndef Heap_h
 #define Heap_h
 
-#include "AllocationSpace.h"
 #include "DFGCodeBlocks.h"
 #include "HandleHeap.h"
 #include "HandleStack.h"
@@ -31,12 +30,16 @@
 #include "MarkedSpace.h"
 #include "SlotVisitor.h"
 #include "WriteBarrierSupport.h"
+#include <wtf/DoublyLinkedList.h>
 #include <wtf/Forward.h>
 #include <wtf/HashCountedSet.h>
 #include <wtf/HashSet.h>
 
+#define COLLECT_ON_EVERY_ALLOCATION 0
+
 namespace JSC {
 
+    class BumpSpace;
     class CodeBlock;
     class GCActivityCallback;
     class GlobalCodeBlock;
@@ -57,7 +60,7 @@ namespace JSC {
     typedef HashCountedSet<const char*> TypeCountSet;
 
     enum OperationInProgress { NoOperation, Allocation, Collection };
-    
+
     // Heap size hint.
     enum HeapSize { SmallHeap, LargeHeap };
 
@@ -65,6 +68,7 @@ namespace JSC {
         WTF_MAKE_NONCOPYABLE(Heap);
     public:
         friend class JIT;
+        friend class MarkStackThreadSharedData;
         static Heap* heap(JSValue); // 0 for immediate values
         static Heap* heap(JSCell*);
 
@@ -78,42 +82,44 @@ namespace JSC {
 
         Heap(JSGlobalData*, HeapSize);
         ~Heap();
-        void destroy(); // JSGlobalData must call destroy() before ~Heap().
+        JS_EXPORT_PRIVATE void destroy(); // JSGlobalData must call destroy() before ~Heap().
 
         JSGlobalData* globalData() const { return m_globalData; }
-        AllocationSpace& objectSpace() { return m_objectSpace; }
+        MarkedSpace& objectSpace() { return m_objectSpace; }
         MachineThreads& machineThreads() { return m_machineThreads; }
 
-        GCActivityCallback* activityCallback();
-        void setActivityCallback(PassOwnPtr<GCActivityCallback>);
+        JS_EXPORT_PRIVATE GCActivityCallback* activityCallback();
+        JS_EXPORT_PRIVATE void setActivityCallback(PassOwnPtr<GCActivityCallback>);
 
         // true if an allocation or collection is in progress
         inline bool isBusy();
         
         MarkedSpace::SizeClass& sizeClassForObject(size_t bytes) { return m_objectSpace.sizeClassFor(bytes); }
         void* allocate(size_t);
+        CheckedBoolean tryAllocateStorage(size_t, void**);
+        CheckedBoolean tryReallocateStorage(void**, size_t, size_t);
 
         typedef void (*Finalizer)(JSCell*);
-        void addFinalizer(JSCell*, Finalizer);
+        JS_EXPORT_PRIVATE void addFinalizer(JSCell*, Finalizer);
 
         void notifyIsSafeToCollect() { m_isSafeToCollect = true; }
-        void collectAllGarbage();
+        JS_EXPORT_PRIVATE void collectAllGarbage();
 
         void reportExtraMemoryCost(size_t cost);
 
-        void protect(JSValue);
-        bool unprotect(JSValue); // True when the protect count drops to 0.
+        JS_EXPORT_PRIVATE void protect(JSValue);
+        JS_EXPORT_PRIVATE bool unprotect(JSValue); // True when the protect count drops to 0.
         
         void jettisonDFGCodeBlock(PassOwnPtr<CodeBlock>);
 
-        size_t size();
-        size_t capacity();
-        size_t objectCount();
-        size_t globalObjectCount();
-        size_t protectedObjectCount();
-        size_t protectedGlobalObjectCount();
-        PassOwnPtr<TypeCountSet> protectedObjectTypeCounts();
-        PassOwnPtr<TypeCountSet> objectTypeCounts();
+        JS_EXPORT_PRIVATE size_t size();
+        JS_EXPORT_PRIVATE size_t capacity();
+        JS_EXPORT_PRIVATE size_t objectCount();
+        JS_EXPORT_PRIVATE size_t globalObjectCount();
+        JS_EXPORT_PRIVATE size_t protectedObjectCount();
+        JS_EXPORT_PRIVATE size_t protectedGlobalObjectCount();
+        JS_EXPORT_PRIVATE PassOwnPtr<TypeCountSet> protectedObjectTypeCounts();
+        JS_EXPORT_PRIVATE PassOwnPtr<TypeCountSet> objectTypeCounts();
 
         void pushTempSortVector(Vector<ValueStringPair>*);
         void popTempSortVector(Vector<ValueStringPair>*);
@@ -129,11 +135,16 @@ namespace JSC {
         void getConservativeRegisterRoots(HashSet<JSCell*>& roots);
 
     private:
+        friend class MarkedSpace;
         friend class MarkedBlock;
-        friend class AllocationSpace;
+        friend class BumpSpace;
         friend class SlotVisitor;
         friend class CodeBlock;
 
+        size_t waterMark();
+        size_t highWaterMark();
+        void setHighWaterMark(size_t);
+
         static const size_t minExtraCost = 256;
         static const size_t maxExtraCost = 1024 * 1024;
         
@@ -141,8 +152,8 @@ namespace JSC {
             virtual void finalize(Handle<Unknown>, void* context);
         };
 
-        bool isValidAllocation(size_t);
-        void reportExtraMemoryCostSlowCase(size_t);
+        JS_EXPORT_PRIVATE bool isValidAllocation(size_t);
+        JS_EXPORT_PRIVATE void reportExtraMemoryCostSlowCase(size_t);
 
         // Call this function before any operation that needs to know which cells
         // in the heap are live. (For example, call this function before
@@ -175,11 +186,14 @@ namespace JSC {
         const HeapSize m_heapSize;
         const size_t m_minBytesPerCycle;
         size_t m_lastFullGCSize;
+        size_t m_waterMark;
+        size_t m_highWaterMark;
         
         OperationInProgress m_operationInProgress;
-        AllocationSpace m_objectSpace;
+        MarkedSpace m_objectSpace;
+        BumpSpace m_storageSpace;
 
-        DoublyLinkedList<MarkedBlock> m_freeBlocks;
+        DoublyLinkedList<HeapBlock> m_freeBlocks;
         size_t m_numberOfFreeBlocks;
         
         ThreadIdentifier m_blockFreeingThread;
@@ -246,6 +260,21 @@ namespace JSC {
         MarkedBlock::blockFor(cell)->setMarked(cell);
     }
 
+    inline size_t Heap::waterMark()
+    {
+        return m_objectSpace.waterMark() + m_storageSpace.totalMemoryUtilized();
+    }
+
+    inline size_t Heap::highWaterMark()
+    {
+        return m_highWaterMark;
+    }
+
+    inline void Heap::setHighWaterMark(size_t newHighWaterMark)
+    {
+        m_highWaterMark = newHighWaterMark;
+    }
+
 #if ENABLE(GGC)
     inline uint8_t* Heap::addressOfCardFor(JSCell* cell)
     {
@@ -308,6 +337,16 @@ namespace JSC {
         ASSERT(isValidAllocation(bytes));
         return m_objectSpace.allocate(bytes);
     }
+    
+    inline CheckedBoolean Heap::tryAllocateStorage(size_t bytes, void** outPtr)
+    {
+        return m_storageSpace.tryAllocate(bytes, outPtr);
+    }
+    
+    inline CheckedBoolean Heap::tryReallocateStorage(void** ptr, size_t oldSize, size_t newSize)
+    {
+        return m_storageSpace.tryReallocate(ptr, oldSize, newSize);
+    }
 
 } // namespace JSC
 
diff --git a/Source/JavaScriptCore/heap/HeapBlock.h b/Source/JavaScriptCore/heap/HeapBlock.h
new file mode 100644
index 000000000..b0ecb2059
--- /dev/null
+++ b/Source/JavaScriptCore/heap/HeapBlock.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+#ifndef HeapBlock_h
+#define HeapBlock_h
+
+#include <wtf/DoublyLinkedList.h>
+#include <wtf/PageAllocationAligned.h>
+
+namespace JSC {
+
+enum AllocationEffort { AllocationCanFail, AllocationMustSucceed };
+
+class HeapBlock : public DoublyLinkedListNode<HeapBlock> {
+public:
+    HeapBlock(PageAllocationAligned& allocation)
+        : DoublyLinkedListNode<HeapBlock>()
+        , m_prev(0)
+        , m_next(0)
+        , m_allocation(allocation)
+    {
+        ASSERT(allocation);
+    }
+
+    HeapBlock* m_prev;
+    HeapBlock* m_next;
+    PageAllocationAligned m_allocation;
+};
+
+} // namespace JSC
+
+#endif    
diff --git a/Source/JavaScriptCore/heap/MachineStackMarker.cpp b/Source/JavaScriptCore/heap/MachineStackMarker.cpp
index f62ee066f..fd828d5de 100644
--- a/Source/JavaScriptCore/heap/MachineStackMarker.cpp
+++ b/Source/JavaScriptCore/heap/MachineStackMarker.cpp
@@ -193,7 +193,7 @@ void MachineThreads::addCurrentThread()
         return;
 
     pthread_setspecific(m_threadSpecific, this);
-    Thread* thread = new Thread(getCurrentPlatformThread(), m_heap->globalData()->stack().origin());
+    Thread* thread = new Thread(getCurrentPlatformThread(), wtfThreadData().stack().origin());
 
     MutexLocker lock(m_registeredThreadsMutex);
 
@@ -257,7 +257,7 @@ void MachineThreads::gatherFromCurrentThread(ConservativeRoots& conservativeRoot
     conservativeRoots.add(registersBegin, registersEnd);
 
     void* stackBegin = stackCurrent;
-    void* stackEnd = m_heap->globalData()->stack().origin();
+    void* stackEnd = wtfThreadData().stack().origin();
     swapIfBackwards(stackBegin, stackEnd);
     conservativeRoots.add(stackBegin, stackEnd);
 }
diff --git a/Source/JavaScriptCore/heap/MachineStackMarker.h b/Source/JavaScriptCore/heap/MachineStackMarker.h
index 69c4537bd..0f5a4c3aa 100644
--- a/Source/JavaScriptCore/heap/MachineStackMarker.h
+++ b/Source/JavaScriptCore/heap/MachineStackMarker.h
@@ -40,7 +40,7 @@ namespace JSC {
         void gatherConservativeRoots(ConservativeRoots&, void* stackCurrent);
 
         void makeUsableFromMultipleThreads();
-        void addCurrentThread(); // Only needs to be called by clients that can use the same heap from multiple threads.
+        JS_EXPORT_PRIVATE void addCurrentThread(); // Only needs to be called by clients that can use the same heap from multiple threads.
 
     private:
         void gatherFromCurrentThread(ConservativeRoots&, void* stackCurrent);
diff --git a/Source/JavaScriptCore/heap/MarkStack.cpp b/Source/JavaScriptCore/heap/MarkStack.cpp
index 02cf328d4..9a3092396 100644
--- a/Source/JavaScriptCore/heap/MarkStack.cpp
+++ b/Source/JavaScriptCore/heap/MarkStack.cpp
@@ -26,6 +26,8 @@
 #include "config.h"
 #include "MarkStack.h"
 
+#include "BumpSpace.h"
+#include "BumpSpaceInlineMethods.h"
 #include "ConservativeRoots.h"
 #include "Heap.h"
 #include "Options.h"
@@ -233,6 +235,7 @@ void* MarkStackThreadSharedData::markingThreadStartFunc(void* shared)
 
 MarkStackThreadSharedData::MarkStackThreadSharedData(JSGlobalData* globalData)
     : m_globalData(globalData)
+    , m_bumpSpace(&globalData->heap.m_storageSpace)
     , m_sharedMarkStack(m_segmentAllocator)
     , m_numberOfActiveParallelMarkers(0)
     , m_parallelMarkersShouldExit(false)
@@ -337,7 +340,7 @@ void SlotVisitor::donateSlow()
 void SlotVisitor::drain()
 {
     ASSERT(m_isInParallelMode);
-    
+   
 #if ENABLE(PARALLEL_GC)
     if (Options::numberOfGCMarkers > 1) {
         while (!m_stack.isEmpty()) {
@@ -398,8 +401,11 @@ void SlotVisitor::drainFromShared(SharedDrainMode sharedDrainMode)
                 // for us to do.
                 while (true) {
                     // Did we reach termination?
-                    if (!m_shared.m_numberOfActiveParallelMarkers && m_shared.m_sharedMarkStack.isEmpty())
+                    if (!m_shared.m_numberOfActiveParallelMarkers && m_shared.m_sharedMarkStack.isEmpty()) {
+                        // Let any sleeping slaves know it's time for them to give their private BumpBlocks back
+                        m_shared.m_markingCondition.broadcast();
                         return;
+                    }
                     
                     // Is there work to be done?
                     if (!m_shared.m_sharedMarkStack.isEmpty())
@@ -415,14 +421,19 @@ void SlotVisitor::drainFromShared(SharedDrainMode sharedDrainMode)
                 if (!m_shared.m_numberOfActiveParallelMarkers && m_shared.m_sharedMarkStack.isEmpty())
                     m_shared.m_markingCondition.broadcast();
                 
-                while (m_shared.m_sharedMarkStack.isEmpty() && !m_shared.m_parallelMarkersShouldExit)
+                while (m_shared.m_sharedMarkStack.isEmpty() && !m_shared.m_parallelMarkersShouldExit) {
+                    if (!m_shared.m_numberOfActiveParallelMarkers && m_shared.m_sharedMarkStack.isEmpty())
+                        doneCopying();
                     m_shared.m_markingCondition.wait(m_shared.m_markingLock);
+                }
                 
                 // Is the VM exiting? If so, exit this thread.
-                if (m_shared.m_parallelMarkersShouldExit)
+                if (m_shared.m_parallelMarkersShouldExit) {
+                    doneCopying();
                     return;
+                }
             }
-            
+           
             m_stack.stealSomeCellsFrom(m_shared.m_sharedMarkStack);
             m_shared.m_numberOfActiveParallelMarkers++;
         }
@@ -445,6 +456,79 @@ void MarkStack::mergeOpaqueRoots()
     m_opaqueRoots.clear();
 }
 
+void SlotVisitor::startCopying()
+{
+    ASSERT(!m_copyBlock);
+    if (!m_shared.m_bumpSpace->borrowBlock(&m_copyBlock))
+        CRASH();
+}    
+
+void* SlotVisitor::allocateNewSpace(void* ptr, size_t bytes)
+{
+    if (BumpSpace::isOversize(bytes)) {
+        m_shared.m_bumpSpace->pin(BumpSpace::oversizeBlockFor(ptr));
+        return 0;
+    }
+
+    if (m_shared.m_bumpSpace->isPinned(ptr))
+        return 0;
+
+    // The only time it's possible to have a null copy block is if we have just started copying.
+    if (!m_copyBlock)
+        startCopying();
+
+    if (!BumpSpace::fitsInBlock(m_copyBlock, bytes)) {
+        // We don't need to lock across these two calls because the master thread won't 
+        // call doneCopying() because this thread is considered active.
+        m_shared.m_bumpSpace->doneFillingBlock(m_copyBlock);
+        if (!m_shared.m_bumpSpace->borrowBlock(&m_copyBlock))
+            CRASH();
+    }
+    return BumpSpace::allocateFromBlock(m_copyBlock, bytes);
+}
+
+void SlotVisitor::copy(void** ptr, size_t bytes)
+{
+    void* newPtr = 0;
+    if (!(newPtr = allocateNewSpace(*ptr, bytes)))
+        return;
+
+    memcpy(newPtr, *ptr, bytes);
+    *ptr = newPtr;
+}
+
+void SlotVisitor::copyAndAppend(void** ptr, size_t bytes, JSValue* values, unsigned length)
+{
+    void* oldPtr = *ptr;
+    void* newPtr = allocateNewSpace(oldPtr, bytes);
+    if (newPtr) {
+        size_t jsValuesOffset = static_cast<size_t>(reinterpret_cast<char*>(values) - static_cast<char*>(oldPtr));
+
+        JSValue* newValues = reinterpret_cast<JSValue*>(static_cast<char*>(newPtr) + jsValuesOffset);
+        for (unsigned i = 0; i < length; i++) {
+            JSValue& value = values[i];
+            newValues[i] = value;
+            if (!value)
+                continue;
+            internalAppend(value);
+        }
+
+        memcpy(newPtr, oldPtr, jsValuesOffset);
+        *ptr = newPtr;
+    } else
+        append(values, length);
+}
+    
+void SlotVisitor::doneCopying()
+{
+    if (!m_copyBlock)
+        return;
+
+    m_shared.m_bumpSpace->doneFillingBlock(m_copyBlock);
+
+    m_copyBlock = 0;
+}
+
 void SlotVisitor::harvestWeakReferences()
 {
     for (WeakReferenceHarvester* current = m_shared.m_weakReferenceHarvesters.head(); current; current = current->next())
diff --git a/Source/JavaScriptCore/heap/MarkStack.h b/Source/JavaScriptCore/heap/MarkStack.h
index 1478011d9..6923cdd8a 100644
--- a/Source/JavaScriptCore/heap/MarkStack.h
+++ b/Source/JavaScriptCore/heap/MarkStack.h
@@ -26,6 +26,7 @@
 #ifndef MarkStack_h
 #define MarkStack_h
 
+#include "BumpSpace.h"
 #include "HandleTypes.h"
 #include "Options.h"
 #include "JSValue.h"
@@ -111,7 +112,7 @@ namespace JSC {
     private:
         MarkStackSegment* m_topSegment;
         
-        void expand();
+        JS_EXPORT_PRIVATE void expand();
         
         MarkStackSegmentAllocator& m_allocator;
 
@@ -181,6 +182,7 @@ namespace JSC {
 #endif
 
         JSGlobalData* m_globalData;
+        BumpSpace* m_bumpSpace;
         
         MarkStackSegmentAllocator m_segmentAllocator;
         
@@ -241,7 +243,7 @@ namespace JSC {
         }
 
     protected:
-        static void validate(JSCell*);
+        JS_EXPORT_PRIVATE static void validate(JSCell*);
 
         void append(JSValue*);
         void append(JSValue*, size_t count);
@@ -250,7 +252,7 @@ namespace JSC {
         void internalAppend(JSCell*);
         void internalAppend(JSValue);
         
-        void mergeOpaqueRoots();
+        JS_EXPORT_PRIVATE void mergeOpaqueRoots();
         
         void mergeOpaqueRootsIfNecessary()
         {
diff --git a/Source/JavaScriptCore/heap/MarkedBlock.cpp b/Source/JavaScriptCore/heap/MarkedBlock.cpp
index 771c9c082..715f25d92 100644
--- a/Source/JavaScriptCore/heap/MarkedBlock.cpp
+++ b/Source/JavaScriptCore/heap/MarkedBlock.cpp
@@ -40,9 +40,9 @@ MarkedBlock* MarkedBlock::create(Heap* heap, size_t cellSize)
     return new (NotNull, allocation.base()) MarkedBlock(allocation, heap, cellSize);
 }
 
-MarkedBlock* MarkedBlock::recycle(MarkedBlock* block, size_t cellSize)
+MarkedBlock* MarkedBlock::recycle(MarkedBlock* block, Heap* heap, size_t cellSize)
 {
-    return new (NotNull, block) MarkedBlock(block->m_allocation, block->m_heap, cellSize);
+    return new (NotNull, block) MarkedBlock(block->m_allocation, heap, cellSize);
 }
 
 void MarkedBlock::destroy(MarkedBlock* block)
@@ -50,13 +50,14 @@ void MarkedBlock::destroy(MarkedBlock* block)
     block->m_allocation.deallocate();
 }
 
-MarkedBlock::MarkedBlock(const PageAllocationAligned& allocation, Heap* heap, size_t cellSize)
-    : m_atomsPerCell((cellSize + atomSize - 1) / atomSize)
+MarkedBlock::MarkedBlock(PageAllocationAligned& allocation, Heap* heap, size_t cellSize)
+    : HeapBlock(allocation)
+    , m_atomsPerCell((cellSize + atomSize - 1) / atomSize)
     , m_endAtom(atomsPerBlock - m_atomsPerCell + 1)
     , m_state(New) // All cells start out unmarked.
-    , m_allocation(allocation)
     , m_heap(heap)
 {
+    ASSERT(heap);
     HEAP_LOG_BLOCK_STATE_TRANSITION(this);
 }
 
diff --git a/Source/JavaScriptCore/heap/MarkedBlock.h b/Source/JavaScriptCore/heap/MarkedBlock.h
index 8c665dd5b..00eb54b1f 100644
--- a/Source/JavaScriptCore/heap/MarkedBlock.h
+++ b/Source/JavaScriptCore/heap/MarkedBlock.h
@@ -23,6 +23,7 @@
 #define MarkedBlock_h
 
 #include "CardSet.h"
+#include "HeapBlock.h"
 
 #include <wtf/Bitmap.h>
 #include <wtf/DoublyLinkedList.h>
@@ -50,7 +51,6 @@ namespace JSC {
 
     typedef uintptr_t Bits;
 
-    static const size_t KB = 1024;
     static const size_t MB = 1024 * 1024;
     
     bool isZapped(const JSCell*);
@@ -63,14 +63,14 @@ namespace JSC {
     // size is equal to the difference between the cell size and the object
     // size.
 
-    class MarkedBlock : public DoublyLinkedListNode<MarkedBlock> {
+    class MarkedBlock : public HeapBlock {
         friend class WTF::DoublyLinkedListNode<MarkedBlock>;
     public:
         // Ensure natural alignment for native types whilst recognizing that the smallest
         // object the heap will commonly allocate is four words.
         static const size_t atomSize = 4 * sizeof(void*);
         static const size_t atomShift = 5;
-        static const size_t blockSize = 16 * KB;
+        static const size_t blockSize = 64 * KB;
         static const size_t blockMask = ~(blockSize - 1); // blockSize must be a power of two.
 
         static const size_t atomsPerBlock = blockSize / atomSize; // ~0.4% overhead
@@ -90,7 +90,7 @@ namespace JSC {
         };
 
         static MarkedBlock* create(Heap*, size_t cellSize);
-        static MarkedBlock* recycle(MarkedBlock*, size_t cellSize);
+        static MarkedBlock* recycle(MarkedBlock*, Heap*, size_t cellSize);
         static void destroy(MarkedBlock*);
 
         static bool isAtomAligned(const void*);
@@ -162,7 +162,7 @@ namespace JSC {
 
         typedef char Atom[atomSize];
 
-        MarkedBlock(const PageAllocationAligned&, Heap*, size_t cellSize);
+        MarkedBlock(PageAllocationAligned&, Heap*, size_t cellSize);
         Atom* atoms();
         size_t atomNumber(const void*);
         void callDestructor(JSCell*);
@@ -180,10 +180,7 @@ namespace JSC {
         WTF::Bitmap<atomsPerBlock, WTF::BitmapNotAtomic> m_marks;
 #endif
         BlockState m_state;
-        PageAllocationAligned m_allocation;
         Heap* m_heap;
-        MarkedBlock* m_prev;
-        MarkedBlock* m_next;
     };
 
     inline size_t MarkedBlock::firstAtom()
diff --git a/Source/JavaScriptCore/heap/MarkedSpace.cpp b/Source/JavaScriptCore/heap/MarkedSpace.cpp
index acbd8acac..fcca188e4 100644
--- a/Source/JavaScriptCore/heap/MarkedSpace.cpp
+++ b/Source/JavaScriptCore/heap/MarkedSpace.cpp
@@ -33,7 +33,6 @@ class Structure;
 MarkedSpace::MarkedSpace(Heap* heap)
     : m_waterMark(0)
     , m_nurseryWaterMark(0)
-    , m_highWaterMark(0)
     , m_heap(heap)
 {
     for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep)
@@ -82,4 +81,192 @@ void MarkedSpace::canonicalizeCellLivenessData()
         sizeClassFor(cellSize).zapFreeList();
 }
 
+inline void* MarkedSpace::tryAllocateHelper(MarkedSpace::SizeClass& sizeClass)
+{
+    MarkedBlock::FreeCell* firstFreeCell = sizeClass.firstFreeCell;
+    if (!firstFreeCell) {
+        for (MarkedBlock*& block = sizeClass.currentBlock; block; block = static_cast<MarkedBlock*>(block->next())) {
+            firstFreeCell = block->sweep(MarkedBlock::SweepToFreeList);
+            if (firstFreeCell)
+                break;
+            m_nurseryWaterMark += block->capacity() - block->size();
+            m_waterMark += block->capacity();
+            block->didConsumeFreeList();
+        }
+
+        if (!firstFreeCell)
+            return 0;
+    }
+
+    ASSERT(firstFreeCell);
+    sizeClass.firstFreeCell = firstFreeCell->next;
+    return firstFreeCell;
+}
+
+inline void* MarkedSpace::tryAllocate(MarkedSpace::SizeClass& sizeClass)
+{
+    m_heap->m_operationInProgress = Allocation;
+    void* result = tryAllocateHelper(sizeClass);
+    m_heap->m_operationInProgress = NoOperation;
+    return result;
+}
+
+void* MarkedSpace::allocateSlowCase(MarkedSpace::SizeClass& sizeClass)
+{
+#if COLLECT_ON_EVERY_ALLOCATION
+    m_heap->collectAllGarbage();
+    ASSERT(m_heap->m_operationInProgress == NoOperation);
+#endif
+    
+    void* result = tryAllocate(sizeClass);
+    
+    if (LIKELY(result != 0))
+        return result;
+    
+    AllocationEffort allocationEffort;
+    
+    if ((
+#if ENABLE(GGC)
+         nurseryWaterMark() < m_heap->m_minBytesPerCycle
+#else
+         m_heap->waterMark() < m_heap->highWaterMark()
+#endif
+         ) || !m_heap->m_isSafeToCollect)
+        allocationEffort = AllocationMustSucceed;
+    else
+        allocationEffort = AllocationCanFail;
+    
+    MarkedBlock* block = allocateBlock(sizeClass.cellSize, allocationEffort);
+    if (block) {
+        addBlock(sizeClass, block);
+        void* result = tryAllocate(sizeClass);
+        ASSERT(result);
+        return result;
+    }
+    
+    m_heap->collect(Heap::DoNotSweep);
+    
+    result = tryAllocate(sizeClass);
+    
+    if (result)
+        return result;
+    
+    ASSERT(m_heap->waterMark() < m_heap->highWaterMark());
+    
+    addBlock(sizeClass, allocateBlock(sizeClass.cellSize, AllocationMustSucceed));
+    
+    result = tryAllocate(sizeClass);
+    ASSERT(result);
+    return result;
+}
+
+MarkedBlock* MarkedSpace::allocateBlock(size_t cellSize, AllocationEffort allocationEffort)
+{
+    MarkedBlock* block;
+    
+    {
+        MutexLocker locker(m_heap->m_freeBlockLock);
+        if (m_heap->m_numberOfFreeBlocks) {
+            block = static_cast<MarkedBlock*>(m_heap->m_freeBlocks.removeHead());
+            ASSERT(block);
+            m_heap->m_numberOfFreeBlocks--;
+        } else
+            block = 0;
+    }
+    if (block)
+        block = MarkedBlock::recycle(block, m_heap, cellSize);
+    else if (allocationEffort == AllocationCanFail)
+        return 0;
+    else
+        block = MarkedBlock::create(m_heap, cellSize);
+    
+    m_blocks.add(block);
+    
+    return block;
+}
+
+void MarkedSpace::freeBlocks(MarkedBlock* head)
+{
+    MarkedBlock* next;
+    for (MarkedBlock* block = head; block; block = next) {
+        next = static_cast<MarkedBlock*>(block->next());
+        
+        m_blocks.remove(block);
+        block->sweep();
+        MutexLocker locker(m_heap->m_freeBlockLock);
+        m_heap->m_freeBlocks.append(block);
+        m_heap->m_numberOfFreeBlocks++;
+    }
+}
+
+class TakeIfUnmarked {
+public:
+    typedef MarkedBlock* ReturnType;
+    
+    TakeIfUnmarked(MarkedSpace*);
+    void operator()(MarkedBlock*);
+    ReturnType returnValue();
+    
+private:
+    MarkedSpace* m_markedSpace;
+    DoublyLinkedList<MarkedBlock> m_empties;
+};
+
+inline TakeIfUnmarked::TakeIfUnmarked(MarkedSpace* newSpace)
+    : m_markedSpace(newSpace)
+{
+}
+
+inline void TakeIfUnmarked::operator()(MarkedBlock* block)
+{
+    if (!block->markCountIsZero())
+        return;
+    
+    m_markedSpace->removeBlock(block);
+    m_empties.append(block);
+}
+
+inline TakeIfUnmarked::ReturnType TakeIfUnmarked::returnValue()
+{
+    return m_empties.head();
+}
+
+void MarkedSpace::shrink()
+{
+    // We record a temporary list of empties to avoid modifying m_blocks while iterating it.
+    TakeIfUnmarked takeIfUnmarked(this);
+    freeBlocks(forEachBlock(takeIfUnmarked));
+}
+
+#if ENABLE(GGC)
+class GatherDirtyCells {
+    WTF_MAKE_NONCOPYABLE(GatherDirtyCells);
+public:
+    typedef void* ReturnType;
+    
+    explicit GatherDirtyCells(MarkedBlock::DirtyCellVector*);
+    void operator()(MarkedBlock*);
+    ReturnType returnValue() { return 0; }
+    
+private:
+    MarkedBlock::DirtyCellVector* m_dirtyCells;
+};
+
+inline GatherDirtyCells::GatherDirtyCells(MarkedBlock::DirtyCellVector* dirtyCells)
+    : m_dirtyCells(dirtyCells)
+{
+}
+
+inline void GatherDirtyCells::operator()(MarkedBlock* block)
+{
+    block->gatherDirtyCells(*m_dirtyCells);
+}
+
+void MarkedSpace::gatherDirtyCells(MarkedBlock::DirtyCellVector& dirtyCells)
+{
+    GatherDirtyCells gatherDirtyCells(&dirtyCells);
+    forEachBlock(gatherDirtyCells);
+}
+#endif
+
 } // namespace JSC
diff --git a/Source/JavaScriptCore/heap/MarkedSpace.h b/Source/JavaScriptCore/heap/MarkedSpace.h
index 751fe2fee..f7d96c774 100644
--- a/Source/JavaScriptCore/heap/MarkedSpace.h
+++ b/Source/JavaScriptCore/heap/MarkedSpace.h
@@ -24,6 +24,7 @@
 
 #include "MachineStackMarker.h"
 #include "MarkedBlock.h"
+#include "MarkedBlockSet.h"
 #include "PageAllocationAligned.h"
 #include <wtf/Bitmap.h>
 #include <wtf/DoublyLinkedList.h>
@@ -54,31 +55,43 @@ public:
 
         MarkedBlock::FreeCell* firstFreeCell;
         MarkedBlock* currentBlock;
-        DoublyLinkedList<MarkedBlock> blockList;
+        DoublyLinkedList<HeapBlock> blockList;
         size_t cellSize;
     };
 
     MarkedSpace(Heap*);
 
     SizeClass& sizeClassFor(size_t);
+    void* allocate(size_t);
     void* allocate(SizeClass&);
     
     void resetAllocator();
 
     void addBlock(SizeClass&, MarkedBlock*);
     void removeBlock(MarkedBlock*);
+    MarkedBlockSet& blocks() { return m_blocks; }
     
     void canonicalizeCellLivenessData();
 
     size_t waterMark();
-    size_t highWaterMark();
     size_t nurseryWaterMark();
-    void setHighWaterMark(size_t);
 
-    template<typename Functor> typename Functor::ReturnType forEachBlock(Functor&); // Safe to remove the current item while iterating.
+    typedef HashSet<MarkedBlock*>::iterator BlockIterator;
+    
+    template<typename Functor> typename Functor::ReturnType forEachCell(Functor&);
+    template<typename Functor> typename Functor::ReturnType forEachCell();
+    template<typename Functor> typename Functor::ReturnType forEachBlock(Functor&);
     template<typename Functor> typename Functor::ReturnType forEachBlock();
+    
+    void shrink();
+    void freeBlocks(MarkedBlock* head);
 
 private:
+    JS_EXPORT_PRIVATE void* allocateSlowCase(SizeClass&);
+    void* tryAllocateHelper(MarkedSpace::SizeClass&);
+    void* tryAllocate(MarkedSpace::SizeClass&);
+    MarkedBlock* allocateBlock(size_t, AllocationEffort);
+
     // [ 32... 256 ]
     static const size_t preciseStep = MarkedBlock::atomSize;
     static const size_t preciseCutoff = 256;
@@ -93,8 +106,8 @@ private:
     FixedArray<SizeClass, impreciseCount> m_impreciseSizeClasses;
     size_t m_waterMark;
     size_t m_nurseryWaterMark;
-    size_t m_highWaterMark;
     Heap* m_heap;
+    MarkedBlockSet m_blocks;
 };
 
 inline size_t MarkedSpace::waterMark()
@@ -102,19 +115,25 @@ inline size_t MarkedSpace::waterMark()
     return m_waterMark;
 }
 
-inline size_t MarkedSpace::highWaterMark()
+inline size_t MarkedSpace::nurseryWaterMark()
 {
-    return m_highWaterMark;
+    return m_nurseryWaterMark;
 }
 
-inline size_t MarkedSpace::nurseryWaterMark()
+template<typename Functor> inline typename Functor::ReturnType MarkedSpace::forEachCell(Functor& functor)
 {
-    return m_nurseryWaterMark;
+    canonicalizeCellLivenessData();
+
+    BlockIterator end = m_blocks.set().end();
+    for (BlockIterator it = m_blocks.set().begin(); it != end; ++it)
+        (*it)->forEachCell(functor);
+    return functor.returnValue();
 }
 
-inline void MarkedSpace::setHighWaterMark(size_t highWaterMark)
+template<typename Functor> inline typename Functor::ReturnType MarkedSpace::forEachCell()
 {
-    m_highWaterMark = highWaterMark;
+    Functor functor;
+    return forEachCell(functor);
 }
 
 inline MarkedSpace::SizeClass& MarkedSpace::sizeClassFor(size_t bytes)
@@ -125,25 +144,19 @@ inline MarkedSpace::SizeClass& MarkedSpace::sizeClassFor(size_t bytes)
     return m_impreciseSizeClasses[(bytes - 1) / impreciseStep];
 }
 
+inline void* MarkedSpace::allocate(size_t bytes)
+{
+    SizeClass& sizeClass = sizeClassFor(bytes);
+    return allocate(sizeClass);
+}
+
 inline void* MarkedSpace::allocate(SizeClass& sizeClass)
 {
+    // This is a light-weight fast path to cover the most common case.
     MarkedBlock::FreeCell* firstFreeCell = sizeClass.firstFreeCell;
-    if (!firstFreeCell) {
-        for (MarkedBlock*& block = sizeClass.currentBlock; block; block = block->next()) {
-            firstFreeCell = block->sweep(MarkedBlock::SweepToFreeList);
-            if (firstFreeCell)
-                break;
-            m_nurseryWaterMark += block->capacity() - block->size();
-            m_waterMark += block->capacity();
-            block->didConsumeFreeList();
-        }
-
-        if (!firstFreeCell)
-            return 0;
-    }
-
-    ASSERT(firstFreeCell);
-
+    if (UNLIKELY(!firstFreeCell))
+        return allocateSlowCase(sizeClass);
+    
     sizeClass.firstFreeCell = firstFreeCell->next;
     return firstFreeCell;
 }
@@ -152,19 +165,19 @@ template <typename Functor> inline typename Functor::ReturnType MarkedSpace::for
 {
     for (size_t i = 0; i < preciseCount; ++i) {
         SizeClass& sizeClass = m_preciseSizeClasses[i];
-        MarkedBlock* next;
-        for (MarkedBlock* block = sizeClass.blockList.head(); block; block = next) {
+        HeapBlock* next;
+        for (HeapBlock* block = sizeClass.blockList.head(); block; block = next) {
             next = block->next();
-            functor(block);
+            functor(static_cast<MarkedBlock*>(block));
         }
     }
 
     for (size_t i = 0; i < impreciseCount; ++i) {
         SizeClass& sizeClass = m_impreciseSizeClasses[i];
-        MarkedBlock* next;
-        for (MarkedBlock* block = sizeClass.blockList.head(); block; block = next) {
+        HeapBlock* next;
+        for (HeapBlock* block = sizeClass.blockList.head(); block; block = next) {
             next = block->next();
-            functor(block);
+            functor(static_cast<MarkedBlock*>(block));
         }
     }
 
@@ -186,7 +199,7 @@ inline MarkedSpace::SizeClass::SizeClass()
 
 inline void MarkedSpace::SizeClass::resetAllocator()
 {
-    currentBlock = blockList.head();
+    currentBlock = static_cast<MarkedBlock*>(blockList.head());
 }
 
 inline void MarkedSpace::SizeClass::zapFreeList()
diff --git a/Source/JavaScriptCore/heap/SlotVisitor.h b/Source/JavaScriptCore/heap/SlotVisitor.h
index 142d8ca49..e49a9a637 100644
--- a/Source/JavaScriptCore/heap/SlotVisitor.h
+++ b/Source/JavaScriptCore/heap/SlotVisitor.h
@@ -26,6 +26,7 @@
 #ifndef SlotVisitor_h
 #define SlotVisitor_h
 
+#include "BumpSpace.h"
 #include "MarkStack.h"
 
 namespace JSC {
@@ -59,8 +60,15 @@ public:
 
     void harvestWeakReferences();
     void finalizeUnconditionalFinalizers();
+
+    void startCopying();
+    void copy(void**, size_t);
+    void copyAndAppend(void**, size_t, JSValue*, unsigned);
+    void doneCopying(); 
         
 private:
+    void* allocateNewSpace(void*, size_t);
+
     void donateSlow();
     
     void donateKnownParallel()
@@ -69,10 +77,13 @@ private:
             return;
         donateSlow();
     }
+    
+    BumpBlock* m_copyBlock;
 };
 
 inline SlotVisitor::SlotVisitor(MarkStackThreadSharedData& shared)
     : MarkStack(shared)
+    , m_copyBlock(0)
 {
 }
 
diff --git a/Source/JavaScriptCore/heap/TinyBloomFilter.h b/Source/JavaScriptCore/heap/TinyBloomFilter.h
index 82b586309..a75ce8ce5 100644
--- a/Source/JavaScriptCore/heap/TinyBloomFilter.h
+++ b/Source/JavaScriptCore/heap/TinyBloomFilter.h
@@ -36,6 +36,7 @@ public:
 
     void add(Bits);
     bool ruleOut(Bits) const; // True for 0.
+    void reset();
 
 private:
     Bits m_bits;
@@ -62,6 +63,11 @@ inline bool TinyBloomFilter::ruleOut(Bits bits) const
     return false;
 }
 
+inline void TinyBloomFilter::reset()
+{
+    m_bits = 0;
+}
+
 } // namespace JSC
 
 #endif // TinyBloomFilter_h
diff --git a/Source/JavaScriptCore/heap/VTableSpectrum.h b/Source/JavaScriptCore/heap/VTableSpectrum.h
index 8a9737e9b..a50a04f47 100644
--- a/Source/JavaScriptCore/heap/VTableSpectrum.h
+++ b/Source/JavaScriptCore/heap/VTableSpectrum.h
@@ -39,7 +39,7 @@ public:
     ~VTableSpectrum();
     
     void countVPtr(void*);
-    void count(JSCell*);
+    JS_EXPORT_PRIVATE void count(JSCell*);
     
     void dump(FILE* output, const char* comment);
 };
diff --git a/Source/JavaScriptCore/heap/WriteBarrierSupport.h b/Source/JavaScriptCore/heap/WriteBarrierSupport.h
index 00b9bb97f..5d7d2f6fe 100644
--- a/Source/JavaScriptCore/heap/WriteBarrierSupport.h
+++ b/Source/JavaScriptCore/heap/WriteBarrierSupport.h
@@ -81,8 +81,8 @@ public:
     }
 #else
     // These are necessary to work around not having conditional exports.
-    static char usesWithBarrierFromCpp;
-    static char usesWithoutBarrierFromCpp;
+    JS_EXPORTDATA static char usesWithBarrierFromCpp;
+    JS_EXPORTDATA static char usesWithoutBarrierFromCpp;
 #endif // ENABLE(WRITE_BARRIER_PROFILING)
 
     static void countWriteBarrier()