9 files changed, 59 insertions, 67 deletions

diff --git a/Source/JavaScriptCore/heap/CopiedSpace.h b/Source/JavaScriptCore/heap/CopiedSpace.h
index ec394ca8f..e8a4f8724 100644
--- a/Source/JavaScriptCore/heap/CopiedSpace.h
+++ b/Source/JavaScriptCore/heap/CopiedSpace.h
@@ -81,7 +81,7 @@ private:
     static bool isOversize(size_t);
     static CopiedBlock* oversizeBlockFor(void* ptr);
 
-    CheckedBoolean tryAllocateSlowCase(size_t, void**);
+    JS_EXPORT_PRIVATE CheckedBoolean tryAllocateSlowCase(size_t, void**);
     CheckedBoolean tryAllocateOversize(size_t, void**);
     CheckedBoolean tryReallocateOversize(void**, size_t, size_t);
     
diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp
index 4ab8ccb31..c7fb8c015 100644
--- a/Source/JavaScriptCore/heap/Heap.cpp
+++ b/Source/JavaScriptCore/heap/Heap.cpp
@@ -618,7 +618,7 @@ size_t Heap::protectedGlobalObjectCount()
 
 size_t Heap::globalObjectCount()
 {
-    return m_objectSpace.forEachCell<CountIfGlobalObject>();
+    return m_objectSpace.forEachLiveCell<CountIfGlobalObject>();
 }
 
 size_t Heap::protectedObjectCount()
@@ -633,7 +633,7 @@ PassOwnPtr<TypeCountSet> Heap::protectedObjectTypeCounts()
 
 PassOwnPtr<TypeCountSet> Heap::objectTypeCounts()
 {
-    return m_objectSpace.forEachCell<RecordType>();
+    return m_objectSpace.forEachLiveCell<RecordType>();
 }
 
 void Heap::deleteAllCompiledCode()
@@ -801,14 +801,11 @@ void Heap::didAllocate(size_t bytes)
     m_bytesAllocated += bytes;
 }
 
-bool Heap::isValidAllocation(size_t bytes)
+bool Heap::isValidAllocation(size_t)
 {
     if (!isValidThreadState(m_globalData))
         return false;
 
-    if (bytes > MarkedSpace::maxCellSize)
-        return false;
-
     if (m_operationInProgress != NoOperation)
         return false;
     
diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h
index e48386791..c9bbec1f6 100644
--- a/Source/JavaScriptCore/heap/Heap.h
+++ b/Source/JavaScriptCore/heap/Heap.h
@@ -182,6 +182,7 @@ namespace JSC {
         friend class CopiedSpace;
         friend class SlotVisitor;
         template<typename T> friend void* allocateCell(Heap&);
+        template<typename T> friend void* allocateCell(Heap&, size_t);
 
         void* allocateWithDestructor(size_t);
         void* allocateWithoutDestructor(size_t);
diff --git a/Source/JavaScriptCore/heap/MarkedAllocator.cpp b/Source/JavaScriptCore/heap/MarkedAllocator.cpp
index ab37ead4c..8a7d02e21 100644
--- a/Source/JavaScriptCore/heap/MarkedAllocator.cpp
+++ b/Source/JavaScriptCore/heap/MarkedAllocator.cpp
@@ -49,7 +49,7 @@ inline void* MarkedAllocator::tryAllocateHelper(size_t bytes)
             }
 
             if (bytes > block->cellSize()) {
-                block->zapFreeList(freeList);
+                block->canonicalizeCellLivenessData(freeList);
                 continue;
             }
 
diff --git a/Source/JavaScriptCore/heap/MarkedAllocator.h b/Source/JavaScriptCore/heap/MarkedAllocator.h
index 7273c13e4..f9cb6ae52 100644
--- a/Source/JavaScriptCore/heap/MarkedAllocator.h
+++ b/Source/JavaScriptCore/heap/MarkedAllocator.h
@@ -21,7 +21,7 @@ class MarkedAllocator {
 public:
     MarkedAllocator();
     void reset();
-    void zapFreeList();
+    void canonicalizeCellLivenessData();
     size_t cellSize() { return m_cellSize; }
     bool cellsNeedDestruction() { return m_cellsNeedDestruction; }
     bool onlyContainsStructures() { return m_onlyContainsStructures; }
@@ -92,14 +92,14 @@ inline void MarkedAllocator::reset()
     m_blocksToSweep = m_blockList.head();
 }
 
-inline void MarkedAllocator::zapFreeList()
+inline void MarkedAllocator::canonicalizeCellLivenessData()
 {
     if (!m_currentBlock) {
         ASSERT(!m_freeList.head);
         return;
     }
     
-    m_currentBlock->zapFreeList(m_freeList);
+    m_currentBlock->canonicalizeCellLivenessData(m_freeList);
     m_currentBlock = 0;
     m_freeList = MarkedBlock::FreeList();
 }
diff --git a/Source/JavaScriptCore/heap/MarkedBlock.cpp b/Source/JavaScriptCore/heap/MarkedBlock.cpp
index b0f3b88d5..c345080fe 100644
--- a/Source/JavaScriptCore/heap/MarkedBlock.cpp
+++ b/Source/JavaScriptCore/heap/MarkedBlock.cpp
@@ -81,8 +81,6 @@ MarkedBlock::FreeList MarkedBlock::specializedSweep()
             continue;
 
         JSCell* cell = reinterpret_cast_ptr<JSCell*>(&atoms()[i]);
-        if (blockState == Zapped && !cell->isZapped())
-            continue;
 
         if (destructorCallNeeded && blockState != New)
             callDestructor(cell);
@@ -95,7 +93,7 @@ MarkedBlock::FreeList MarkedBlock::specializedSweep()
         }
     }
 
-    m_state = ((sweepMode == SweepToFreeList) ? FreeListed : Zapped);
+    m_state = ((sweepMode == SweepToFreeList) ? FreeListed : Marked);
     return FreeList(head, count * cellSize());
 }
 
@@ -132,18 +130,21 @@ MarkedBlock::FreeList MarkedBlock::sweepHelper(SweepMode sweepMode)
         return sweepMode == SweepToFreeList
             ? specializedSweep<Marked, SweepToFreeList, destructorCallNeeded>()
             : specializedSweep<Marked, SweepOnly, destructorCallNeeded>();
-    case Zapped:
-        ASSERT(!m_onlyContainsStructures || heap()->isSafeToSweepStructures());
-        return sweepMode == SweepToFreeList
-            ? specializedSweep<Zapped, SweepToFreeList, destructorCallNeeded>()
-            : specializedSweep<Zapped, SweepOnly, destructorCallNeeded>();
     }
 
     ASSERT_NOT_REACHED();
     return FreeList();
 }
 
-void MarkedBlock::zapFreeList(const FreeList& freeList)
+class SetAllMarksFunctor : public MarkedBlock::VoidFunctor {
+public:
+    void operator()(JSCell* cell)
+    {
+        MarkedBlock::blockFor(cell)->setMarked(cell);
+    }
+};
+
+void MarkedBlock::canonicalizeCellLivenessData(const FreeList& freeList)
 {
     HEAP_LOG_BLOCK_STATE_TRANSITION(this);
     FreeCell* head = freeList.head;
@@ -156,40 +157,26 @@ void MarkedBlock::zapFreeList(const FreeList& freeList)
         // Hence if the block is Marked we need to leave it Marked.
         
         ASSERT(!head);
-        
-        return;
-    }
-    
-    if (m_state == Zapped) {
-        // If the block is in the Zapped state then we know that someone already
-        // zapped it for us. This could not have happened during a GC, but might
-        // be the result of someone having done a GC scan to perform some operation
-        // over all live objects (or all live blocks). It also means that somebody
-        // had allocated in this block since the last GC, swept all dead objects
-        // onto the free list, left the block in the FreeListed state, then the heap
-        // scan happened, and canonicalized the block, leading to all dead objects
-        // being zapped. Therefore, it is safe for us to simply do nothing, since
-        // dead objects will have 0 in their vtables and live objects will have
-        // non-zero vtables, which is consistent with the block being zapped.
-        
-        ASSERT(!head);
-        
         return;
     }
-    
+   
     ASSERT(m_state == FreeListed);
     
     // Roll back to a coherent state for Heap introspection. Cells newly
     // allocated from our free list are not currently marked, so we need another
-    // way to tell what's live vs dead. We use zapping for that.
+    // way to tell what's live vs dead. 
     
+    SetAllMarksFunctor functor;
+    forEachCell(functor);
+
     FreeCell* next;
     for (FreeCell* current = head; current; current = next) {
         next = current->next;
         reinterpret_cast<JSCell*>(current)->zap();
+        clearMarked(current);
     }
     
-    m_state = Zapped;
+    m_state = Marked;
 }
 
 } // namespace JSC
diff --git a/Source/JavaScriptCore/heap/MarkedBlock.h b/Source/JavaScriptCore/heap/MarkedBlock.h
index 8c03d5b4f..4b2a5fd53 100644
--- a/Source/JavaScriptCore/heap/MarkedBlock.h
+++ b/Source/JavaScriptCore/heap/MarkedBlock.h
@@ -136,7 +136,7 @@ namespace JSC {
         // cell liveness data. To restore accurate cell liveness data, call one
         // of these functions:
         void didConsumeFreeList(); // Call this once you've allocated all the items in the free list.
-        void zapFreeList(const FreeList&); // Call this to undo the free list.
+        void canonicalizeCellLivenessData(const FreeList&);
 
         void clearMarks();
         size_t markCount();
@@ -154,7 +154,8 @@ namespace JSC {
         bool isLive(const JSCell*);
         bool isLiveCell(const void*);
         void setMarked(const void*);
-        
+        void clearMarked(const void*);
+
         bool needsSweeping();
 
 #if ENABLE(GGC)
@@ -186,12 +187,13 @@ namespace JSC {
 #endif
 
         template <typename Functor> void forEachCell(Functor&);
+        template <typename Functor> void forEachLiveCell(Functor&);
         template <typename Functor> void forEachDeadCell(Functor&);
 
     private:
         static const size_t atomAlignmentMask = atomSize - 1; // atomSize must be a power of two.
 
-        enum BlockState { New, FreeListed, Allocated, Marked, Zapped };
+        enum BlockState { New, FreeListed, Allocated, Marked };
         template<bool destructorCallNeeded> FreeList sweepHelper(SweepMode = SweepOnly);
 
         typedef char Atom[atomSize];
@@ -364,21 +366,18 @@ namespace JSC {
         m_marks.set(atomNumber(p));
     }
 
+    inline void MarkedBlock::clearMarked(const void* p)
+    {
+        ASSERT(m_marks.get(atomNumber(p)));
+        m_marks.clear(atomNumber(p));
+    }
+
     inline bool MarkedBlock::isLive(const JSCell* cell)
     {
         switch (m_state) {
         case Allocated:
             return true;
-        case Zapped:
-            if (isZapped(cell)) {
-                // Object dead in previous collection, not allocated since previous collection: mark bit should not be set.
-                ASSERT(!m_marks.get(atomNumber(cell)));
-                return false;
-            }
-            
-            // Newly allocated objects: mark bit not set.
-            // Objects that survived prior collection: mark bit set.
-            return true;
+
         case Marked:
             return m_marks.get(atomNumber(cell));
 
@@ -411,6 +410,14 @@ namespace JSC {
     {
         for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
             JSCell* cell = reinterpret_cast_ptr<JSCell*>(&atoms()[i]);
+            functor(cell);
+        }
+    }
+
+    template <typename Functor> inline void MarkedBlock::forEachLiveCell(Functor& functor)
+    {
+        for (size_t i = firstAtom(); i < m_endAtom; i += m_atomsPerCell) {
+            JSCell* cell = reinterpret_cast_ptr<JSCell*>(&atoms()[i]);
             if (!isLive(cell))
                 continue;
 
@@ -431,7 +438,7 @@ namespace JSC {
 
     inline bool MarkedBlock::needsSweeping()
     {
-        return m_state == Marked || m_state == Zapped;
+        return m_state == Marked;
     }
 
 #if ENABLE(GGC)
diff --git a/Source/JavaScriptCore/heap/MarkedSpace.cpp b/Source/JavaScriptCore/heap/MarkedSpace.cpp
index 689e5f9ab..9a823c50b 100644
--- a/Source/JavaScriptCore/heap/MarkedSpace.cpp
+++ b/Source/JavaScriptCore/heap/MarkedSpace.cpp
@@ -146,17 +146,17 @@ void MarkedSpace::reapWeakSets()
 void MarkedSpace::canonicalizeCellLivenessData()
 {
     for (size_t cellSize = preciseStep; cellSize <= preciseCutoff; cellSize += preciseStep) {
-        allocatorFor(cellSize).zapFreeList();
-        destructorAllocatorFor(cellSize).zapFreeList();
+        allocatorFor(cellSize).canonicalizeCellLivenessData();
+        destructorAllocatorFor(cellSize).canonicalizeCellLivenessData();
     }
 
     for (size_t cellSize = impreciseStep; cellSize <= impreciseCutoff; cellSize += impreciseStep) {
-        allocatorFor(cellSize).zapFreeList();
-        destructorAllocatorFor(cellSize).zapFreeList();
+        allocatorFor(cellSize).canonicalizeCellLivenessData();
+        destructorAllocatorFor(cellSize).canonicalizeCellLivenessData();
     }
 
-    m_largeAllocator.zapFreeList();
-    m_structureAllocator.zapFreeList();
+    m_largeAllocator.canonicalizeCellLivenessData();
+    m_structureAllocator.canonicalizeCellLivenessData();
 }
 
 bool MarkedSpace::isPagedOut(double deadline)
diff --git a/Source/JavaScriptCore/heap/MarkedSpace.h b/Source/JavaScriptCore/heap/MarkedSpace.h
index 03679d9d3..e68ba9172 100644
--- a/Source/JavaScriptCore/heap/MarkedSpace.h
+++ b/Source/JavaScriptCore/heap/MarkedSpace.h
@@ -93,8 +93,8 @@ public:
 
     typedef HashSet<MarkedBlock*>::iterator BlockIterator;
     
-    template<typename Functor> typename Functor::ReturnType forEachCell(Functor&);
-    template<typename Functor> typename Functor::ReturnType forEachCell();
+    template<typename Functor> typename Functor::ReturnType forEachLiveCell(Functor&);
+    template<typename Functor> typename Functor::ReturnType forEachLiveCell();
     template<typename Functor> typename Functor::ReturnType forEachBlock(Functor&);
     template<typename Functor> typename Functor::ReturnType forEachBlock();
     
@@ -140,20 +140,20 @@ private:
     MarkedBlockSet m_blocks;
 };
 
-template<typename Functor> inline typename Functor::ReturnType MarkedSpace::forEachCell(Functor& functor)
+template<typename Functor> inline typename Functor::ReturnType MarkedSpace::forEachLiveCell(Functor& functor)
 {
     canonicalizeCellLivenessData();
 
     BlockIterator end = m_blocks.set().end();
     for (BlockIterator it = m_blocks.set().begin(); it != end; ++it)
-        (*it)->forEachCell(functor);
+        (*it)->forEachLiveCell(functor);
     return functor.returnValue();
 }
 
-template<typename Functor> inline typename Functor::ReturnType MarkedSpace::forEachCell()
+template<typename Functor> inline typename Functor::ReturnType MarkedSpace::forEachLiveCell()
 {
     Functor functor;
-    return forEachCell(functor);
+    return forEachLiveCell(functor);
 }
 
 inline MarkedAllocator& MarkedSpace::firstAllocator()