Changeset 52176 in webkit for trunk/JavaScriptCore/runtime

Timestamp:

Dec 15, 2009, 2:40:00 PM (15 years ago)

Author:

[email protected]

Message:

Removed the number heap, replacing it with a one-item free list for
numbers, taking advantage of the fact that two number cells fit inside
the space for one regular cell, and number cells don't require destruction.

Reviewed by Oliver Hunt.

SunSpider says 1.6% faster in JSVALUE32 mode (the only mode that
heap-allocates numbers).

SunSpider says 1.1% faster in JSVALUE32_64 mode. v8 says 0.8% faster
in JSVALUE32_64 mode. 10% speedup on bench-alloc-nonretained.js. 6%
speedup on bench-alloc-retained.js.

There's a lot of formulaic change in this patch, but not much substance.

• JavaScriptCore.exp:
• debugger/Debugger.cpp:

(JSC::Debugger::recompileAllJSFunctions):

• runtime/Collector.cpp:

(JSC::Heap::Heap):
(JSC::Heap::destroy):
(JSC::Heap::allocateBlock):
(JSC::Heap::freeBlock):
(JSC::Heap::freeBlockPtr):
(JSC::Heap::freeBlocks):
(JSC::Heap::recordExtraCost):
(JSC::Heap::allocate):
(JSC::Heap::resizeBlocks):
(JSC::Heap::growBlocks):
(JSC::Heap::shrinkBlocks):
(JSC::Heap::markConservatively):
(JSC::Heap::clearMarkBits):
(JSC::Heap::markedCells):
(JSC::Heap::sweep):
(JSC::Heap::markRoots):
(JSC::Heap::objectCount):
(JSC::Heap::addToStatistics):
(JSC::Heap::statistics):
(JSC::Heap::isBusy):
(JSC::Heap::reset):
(JSC::Heap::collectAllGarbage):
(JSC::Heap::primaryHeapBegin):
(JSC::Heap::primaryHeapEnd):

• runtime/Collector.h:

(JSC::): Removed all code pertaining to the number heap, and changed all
heap template functions and classes to non-template functions and classes.

(JSC::Heap::allocateNumber): A new optimization to replace the number
heap: allocate half-sized number cells in pairs, returning the first
cell and caching the second cell for the next allocation.

• runtime/CollectorHeapIterator.h:

(JSC::LiveObjectIterator::LiveObjectIterator):
(JSC::LiveObjectIterator::operator++):
(JSC::DeadObjectIterator::DeadObjectIterator):
(JSC::DeadObjectIterator::operator++):
(JSC::ObjectIterator::ObjectIterator):
(JSC::ObjectIterator::operator++):

• runtime/JSCell.h:

(JSC::JSCell::isNumber): Removed all code pertaining to the number heap,
and changed all heap template functions and classes to non-template functions
and classes.

Location:

trunk/JavaScriptCore/runtime

Files:

• Collector.cpp (modified) (22 diffs)
• Collector.h (modified) (8 diffs)
• CollectorHeapIterator.h (modified) (6 diffs)
• JSCell.h (modified) (1 diff)

trunk/JavaScriptCore/runtime/Collector.cpp

@@ -170 +170 @@
 #endif // PLATFORM(SYMBIAN)
     
-    memset(&primaryHeap, 0, sizeof(CollectorHeap));
-    allocateBlock<PrimaryHeap>();
-
-    memset(&numberHeap, 0, sizeof(CollectorHeap));
-#if USE(JSVALUE32)
-    allocateBlock<NumberHeap>();
-#endif
+    memset(&m_heap, 0, sizeof(CollectorHeap));
+    allocateBlock();
 }
 
@@ -199 +194 @@
     m_markListSet = 0;
 
-    freeBlocks<PrimaryHeap>();
-    freeBlocks<NumberHeap>();
+    freeBlocks();
 
 #if ENABLE(JSC_MULTIPLE_THREADS)
@@ -219 +213 @@
 }
 
-template <HeapType heapType>
 NEVER_INLINE CollectorBlock* Heap::allocateBlock()
 {
@@ -274 +267 @@
     CollectorBlock* block = reinterpret_cast<CollectorBlock*>(address);
     block->heap = this;
-    block->type = heapType;
-    clearMarkBits<heapType>(block);
-
-    // heapAllocate assumes that it's safe to call a destructor on any cell in the primary heap.
-    if (heapType != NumberHeap) {
-        Structure* dummyMarkableCellStructure = m_globalData->dummyMarkableCellStructure.get();
-        for (size_t i = 0; i < HeapConstants<heapType>::cellsPerBlock; ++i)
-            new (block->cells + i) JSCell(dummyMarkableCellStructure);
-    }
+    clearMarkBits(block);
+
+    Structure* dummyMarkableCellStructure = m_globalData->dummyMarkableCellStructure.get();
+    for (size_t i = 0; i < HeapConstants::cellsPerBlock; ++i)
+        new (block->cells + i) JSCell(dummyMarkableCellStructure);
     
     // Add block to blocks vector.
 
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-    size_t numBlocks = heap.numBlocks;
-    if (heap.usedBlocks == numBlocks) {
+    size_t numBlocks = m_heap.numBlocks;
+    if (m_heap.usedBlocks == numBlocks) {
         static const size_t maxNumBlocks = ULONG_MAX / sizeof(CollectorBlock*) / GROWTH_FACTOR;
         if (numBlocks > maxNumBlocks)
             CRASH();
         numBlocks = max(MIN_ARRAY_SIZE, numBlocks * GROWTH_FACTOR);
-        heap.numBlocks = numBlocks;
-        heap.blocks = static_cast<CollectorBlock**>(fastRealloc(heap.blocks, numBlocks * sizeof(CollectorBlock*)));
-    }
-    heap.blocks[heap.usedBlocks++] = block;
+        m_heap.numBlocks = numBlocks;
+        m_heap.blocks = static_cast<CollectorBlock**>(fastRealloc(m_heap.blocks, numBlocks * sizeof(CollectorBlock*)));
+    }
+    m_heap.blocks[m_heap.usedBlocks++] = block;
 
     return block;
 }
 
-template <HeapType heapType>
 NEVER_INLINE void Heap::freeBlock(size_t block)
 {
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-
-    if (heapType != NumberHeap) {
-        ObjectIterator<heapType> it(heap, block);
-        ObjectIterator<heapType> end(heap, block + 1);
-        for ( ; it != end; ++it)
-            (*it)->~JSCell();
-    }
-    freeBlock(heap.blocks[block]);
+    ObjectIterator it(m_heap, block);
+    ObjectIterator end(m_heap, block + 1);
+    for ( ; it != end; ++it)
+        (*it)->~JSCell();
+    freeBlockPtr(m_heap.blocks[block]);
 
     // swap with the last block so we compact as we go
-    heap.blocks[block] = heap.blocks[heap.usedBlocks - 1];
-    heap.usedBlocks--;
-
-    if (heap.numBlocks > MIN_ARRAY_SIZE && heap.usedBlocks < heap.numBlocks / LOW_WATER_FACTOR) {
-        heap.numBlocks = heap.numBlocks / GROWTH_FACTOR; 
-        heap.blocks = static_cast<CollectorBlock**>(fastRealloc(heap.blocks, heap.numBlocks * sizeof(CollectorBlock*)));
-    }
-}
-
-NEVER_INLINE void Heap::freeBlock(CollectorBlock* block)
+    m_heap.blocks[block] = m_heap.blocks[m_heap.usedBlocks - 1];
+    m_heap.usedBlocks--;
+
+    if (m_heap.numBlocks > MIN_ARRAY_SIZE && m_heap.usedBlocks < m_heap.numBlocks / LOW_WATER_FACTOR) {
+        m_heap.numBlocks = m_heap.numBlocks / GROWTH_FACTOR; 
+        m_heap.blocks = static_cast<CollectorBlock**>(fastRealloc(m_heap.blocks, m_heap.numBlocks * sizeof(CollectorBlock*)));
+    }
+}
+
+NEVER_INLINE void Heap::freeBlockPtr(CollectorBlock* block)
 {
 #if PLATFORM(DARWIN)    
@@ -345 +328 @@
 }
 
-template <HeapType heapType>
 void Heap::freeBlocks()
 {
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-
-    while (heap.usedBlocks)
-        freeBlock<heapType>(0);
-    fastFree(heap.blocks);
-    memset(&heap, 0, sizeof(CollectorHeap));
+    while (m_heap.usedBlocks)
+        freeBlock(0);
+    fastFree(m_heap.blocks);
+    memset(&m_heap, 0, sizeof(CollectorHeap));
 }
 
@@ -368 +348 @@
     // if a large value survives one garbage collection, there is not much point to
     // collecting more frequently as long as it stays alive.
-    // NOTE: we target the primaryHeap unconditionally as JSNumber doesn't modify cost 
-
-    if (primaryHeap.extraCost > maxExtraCost && primaryHeap.extraCost > primaryHeap.usedBlocks * BLOCK_SIZE / 2) {
+
+    if (m_heap.extraCost > maxExtraCost && m_heap.extraCost > m_heap.usedBlocks * BLOCK_SIZE / 2) {
         // If the last iteration through the heap deallocated blocks, we need
         // to clean up remaining garbage before marking. Otherwise, the conservative
         // marking mechanism might follow a pointer to unmapped memory.
-        if (primaryHeap.didShrink)
-            sweep<PrimaryHeap>();
+        if (m_heap.didShrink)
+            sweep();
         reset();
     }
-    primaryHeap.extraCost += cost;
-}
-
-template <HeapType heapType> ALWAYS_INLINE void* Heap::heapAllocate(size_t s)
-{
-    typedef typename HeapConstants<heapType>::Block Block;
-    typedef typename HeapConstants<heapType>::Cell Cell;
+    m_heap.extraCost += cost;
+}
+
+void* Heap::allocate(size_t s)
+{
+    typedef HeapConstants::Block Block;
+    typedef HeapConstants::Cell Cell;
     
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-
     ASSERT(JSLock::lockCount() > 0);
     ASSERT(JSLock::currentThreadIsHoldingLock());
-    ASSERT_UNUSED(s, s <= HeapConstants<heapType>::cellSize);
-
-    ASSERT(heap.operationInProgress == NoOperation);
-    ASSERT(heapType == PrimaryHeap || heap.extraCost == 0);
+    ASSERT_UNUSED(s, s <= HeapConstants::cellSize);
+
+    ASSERT(m_heap.operationInProgress == NoOperation);
 
 #if COLLECT_ON_EVERY_ALLOCATION
     collectAllGarbage();
-    ASSERT(heap.operationInProgress == NoOperation);
+    ASSERT(m_heap.operationInProgress == NoOperation);
 #endif
 
@@ -405 +381 @@
 
     do {
-        ASSERT(heap.nextBlock < heap.usedBlocks);
-        Block* block = reinterpret_cast<Block*>(heap.blocks[heap.nextBlock]);
+        ASSERT(m_heap.nextBlock < m_heap.usedBlocks);
+        Block* block = reinterpret_cast<Block*>(m_heap.blocks[m_heap.nextBlock]);
         do {
-            ASSERT(heap.nextCell < HeapConstants<heapType>::cellsPerBlock);
-            if (!block->marked.get(heap.nextCell >> HeapConstants<heapType>::bitmapShift)) { // Always false for the last cell in the block
-                Cell* cell = block->cells + heap.nextCell;
-                if (heapType != NumberHeap) {
-                    heap.operationInProgress = Allocation;
-                    JSCell* imp = reinterpret_cast<JSCell*>(cell);
-                    imp->~JSCell();
-                    heap.operationInProgress = NoOperation;
-                }
-                ++heap.nextCell;
+            ASSERT(m_heap.nextCell < HeapConstants::cellsPerBlock);
+            if (!block->marked.get(m_heap.nextCell)) { // Always false for the last cell in the block
+                Cell* cell = block->cells + m_heap.nextCell;
+
+                m_heap.operationInProgress = Allocation;
+                JSCell* imp = reinterpret_cast<JSCell*>(cell);
+                imp->~JSCell();
+                m_heap.operationInProgress = NoOperation;
+
+                ++m_heap.nextCell;
                 return cell;
             }
-        } while (++heap.nextCell != HeapConstants<heapType>::cellsPerBlock);
-        heap.nextCell = 0;
-    } while (++heap.nextBlock != heap.usedBlocks);
+        } while (++m_heap.nextCell != HeapConstants::cellsPerBlock);
+        m_heap.nextCell = 0;
+    } while (++m_heap.nextBlock != m_heap.usedBlocks);
 
     // Slow case: reached the end of the heap. Mark live objects and start over.
@@ -430 +406 @@
 }
 
-template <HeapType heapType>
 void Heap::resizeBlocks()
 {
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-
-    heap.didShrink = false;
-
-    size_t usedCellCount = markedCells<heapType>();
+    m_heap.didShrink = false;
+
+    size_t usedCellCount = markedCells();
     size_t minCellCount = usedCellCount + max(ALLOCATIONS_PER_COLLECTION, usedCellCount);
-    size_t minBlockCount = (minCellCount + HeapConstants<heapType>::cellsPerBlock - 1) / HeapConstants<heapType>::cellsPerBlock;
+    size_t minBlockCount = (minCellCount + HeapConstants::cellsPerBlock - 1) / HeapConstants::cellsPerBlock;
 
     size_t maxCellCount = 1.25f * minCellCount;
-    size_t maxBlockCount = (maxCellCount + HeapConstants<heapType>::cellsPerBlock - 1) / HeapConstants<heapType>::cellsPerBlock;
-
-    if (heap.usedBlocks < minBlockCount)
-        growBlocks<heapType>(minBlockCount);
-    else if (heap.usedBlocks > maxBlockCount)
-        shrinkBlocks<heapType>(maxBlockCount);
-}
-
-template <HeapType heapType> 
+    size_t maxBlockCount = (maxCellCount + HeapConstants::cellsPerBlock - 1) / HeapConstants::cellsPerBlock;
+
+    if (m_heap.usedBlocks < minBlockCount)
+        growBlocks(minBlockCount);
+    else if (m_heap.usedBlocks > maxBlockCount)
+        shrinkBlocks(maxBlockCount);
+}
+
 void Heap::growBlocks(size_t neededBlocks)
 {
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-    ASSERT(heap.usedBlocks < neededBlocks);
-    while (heap.usedBlocks < neededBlocks)
-        allocateBlock<heapType>();
-}
-
-template <HeapType heapType> 
+    ASSERT(m_heap.usedBlocks < neededBlocks);
+    while (m_heap.usedBlocks < neededBlocks)
+        allocateBlock();
+}
+
 void Heap::shrinkBlocks(size_t neededBlocks)
 {
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-    ASSERT(heap.usedBlocks > neededBlocks);
+    ASSERT(m_heap.usedBlocks > neededBlocks);
     
     // Clear the always-on last bit, so isEmpty() isn't fooled by it.
-    for (size_t i = 0; i < heap.usedBlocks; ++i)
-        heap.blocks[i]->marked.clear((HeapConstants<heapType>::cellsPerBlock - 1) >> HeapConstants<heapType>::bitmapShift);
-
-    for (size_t i = 0; i != heap.usedBlocks && heap.usedBlocks != neededBlocks; ) {
-        if (heap.blocks[i]->marked.isEmpty()) {
-            freeBlock<heapType>(i);
-            heap.didShrink = true;
+    for (size_t i = 0; i < m_heap.usedBlocks; ++i)
+        m_heap.blocks[i]->marked.clear(HeapConstants::cellsPerBlock - 1);
+
+    for (size_t i = 0; i != m_heap.usedBlocks && m_heap.usedBlocks != neededBlocks; ) {
+        if (m_heap.blocks[i]->marked.isEmpty()) {
+            freeBlock(i);
+            m_heap.didShrink = true;
         } else
             ++i;
@@ -478 +447 @@
 
     // Reset the always-on last bit.
-    for (size_t i = 0; i < heap.usedBlocks; ++i)
-        heap.blocks[i]->marked.set((HeapConstants<heapType>::cellsPerBlock - 1) >> HeapConstants<heapType>::bitmapShift);
-}
-
-void* Heap::allocate(size_t s)
-{
-    return heapAllocate<PrimaryHeap>(s);
-}
-
-void* Heap::allocateNumber(size_t s)
-{
-    return heapAllocate<NumberHeap>(s);
+    for (size_t i = 0; i < m_heap.usedBlocks; ++i)
+        m_heap.blocks[i]->marked.set(HeapConstants::cellsPerBlock - 1);
 }
 
@@ -765 +724 @@
     return isCellAligned(p) && p;
 }
-#endif
+#endif // USE(JSVALUE32)
 
 void Heap::markConservatively(MarkStack& markStack, void* start, void* end)
@@ -782 +741 @@
     char** e = static_cast<char**>(end);
 
-    CollectorBlock** primaryBlocks = primaryHeap.blocks;
-#if USE(JSVALUE32)
-    CollectorBlock** numberBlocks = numberHeap.blocks;
-#endif
-
+    CollectorBlock** blocks = m_heap.blocks;
     while (p != e) {
         char* x = *p++;
         if (isPossibleCell(x)) {
-            size_t usedPrimaryBlocks;
+            size_t usedBlocks;
             uintptr_t xAsBits = reinterpret_cast<uintptr_t>(x);
             xAsBits &= CELL_ALIGN_MASK;
@@ -800 +755 @@
 
             CollectorBlock* blockAddr = reinterpret_cast<CollectorBlock*>(xAsBits - offset);
-#if USE(JSVALUE32)
-            // Mark the the number heap, we can mark these Cells directly to avoid the virtual call cost
-            size_t usedNumberBlocks = numberHeap.usedBlocks;
-            for (size_t block = 0; block < usedNumberBlocks; block++) {
-                if (numberBlocks[block] != blockAddr)
-                    continue;
-                Heap::markCell(reinterpret_cast<JSCell*>(xAsBits));
-                goto endMarkLoop;
-            }
-#endif
-
-            // Mark the primary heap
-            usedPrimaryBlocks = primaryHeap.usedBlocks;
-            for (size_t block = 0; block < usedPrimaryBlocks; block++) {
-                if (primaryBlocks[block] != blockAddr)
+            usedBlocks = m_heap.usedBlocks;
+            for (size_t block = 0; block < usedBlocks; block++) {
+                if (blocks[block] != blockAddr)
                     continue;
                 markStack.append(reinterpret_cast<JSCell*>(xAsBits));
                 markStack.drain();
             }
-#if USE(JSVALUE32)
-        endMarkLoop:
-            ;
-#endif
         }
     }
@@ -1064 +1003 @@
 }
 
-template <HeapType heapType> 
 void Heap::clearMarkBits()
 {
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-    for (size_t i = 0; i < heap.usedBlocks; ++i)
-        clearMarkBits<heapType>(heap.blocks[i]);
-}
-
-template <HeapType heapType> 
+    for (size_t i = 0; i < m_heap.usedBlocks; ++i)
+        clearMarkBits(m_heap.blocks[i]);
+}
+
 void Heap::clearMarkBits(CollectorBlock* block)
 {
-    // heapAllocate assumes that the last cell in every block is marked.
+    // allocate assumes that the last cell in every block is marked.
     block->marked.clearAll();
-    block->marked.set((HeapConstants<heapType>::cellsPerBlock - 1) >> HeapConstants<heapType>::bitmapShift);
-}
-
-template <HeapType heapType> 
+    block->marked.set(HeapConstants::cellsPerBlock - 1);
+}
+
 size_t Heap::markedCells(size_t startBlock, size_t startCell) const
 {
-    const CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-    ASSERT(startBlock <= heap.usedBlocks);
-    ASSERT(startCell < HeapConstants<heapType>::cellsPerBlock);
-
-    if (startBlock >= heap.usedBlocks)
+    ASSERT(startBlock <= m_heap.usedBlocks);
+    ASSERT(startCell < HeapConstants::cellsPerBlock);
+
+    if (startBlock >= m_heap.usedBlocks)
         return 0;
 
     size_t result = 0;
-    result += heap.blocks[startBlock]->marked.count(startCell);
-    for (size_t i = startBlock + 1; i < heap.usedBlocks; ++i)
-        result += heap.blocks[i]->marked.count();
+    result += m_heap.blocks[startBlock]->marked.count(startCell);
+    for (size_t i = startBlock + 1; i < m_heap.usedBlocks; ++i)
+        result += m_heap.blocks[i]->marked.count();
 
     return result;
 }
 
-template <HeapType heapType> 
 void Heap::sweep()
 {
-    ASSERT(heapType != NumberHeap); // The number heap does not contain meaningful destructors.
-
-    CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-
-    ASSERT(heap.operationInProgress == NoOperation);
-    if (heap.operationInProgress != NoOperation)
+    ASSERT(m_heap.operationInProgress == NoOperation);
+    if (m_heap.operationInProgress != NoOperation)
         CRASH();
-    heap.operationInProgress = Collection;
+    m_heap.operationInProgress = Collection;
     
 #if !ENABLE(JSC_ZOMBIES)
@@ -1114 +1043 @@
 #endif
 
-    DeadObjectIterator<heapType> it(heap, heap.nextBlock, heap.nextCell);
-    DeadObjectIterator<heapType> end(heap, heap.usedBlocks);
+    DeadObjectIterator it(m_heap, m_heap.nextBlock, m_heap.nextCell);
+    DeadObjectIterator end(m_heap, m_heap.usedBlocks);
     for ( ; it != end; ++it) {
         JSCell* cell = *it;
@@ -1132 +1061 @@
     }
 
-    heap.operationInProgress = NoOperation;
+    m_heap.operationInProgress = NoOperation;
 }
 
@@ -1144 +1073 @@
 #endif
 
-    ASSERT((primaryHeap.operationInProgress == NoOperation) & (numberHeap.operationInProgress == NoOperation));
-    if (!((primaryHeap.operationInProgress == NoOperation) & (numberHeap.operationInProgress == NoOperation)))
+    ASSERT(m_heap.operationInProgress == NoOperation);
+    if (m_heap.operationInProgress != NoOperation)
         CRASH();
 
-    primaryHeap.operationInProgress = Collection;
-    numberHeap.operationInProgress = Collection;
+    m_heap.operationInProgress = Collection;
 
     MarkStack& markStack = m_globalData->markStack;
 
     // Reset mark bits.
-    clearMarkBits<PrimaryHeap>();
-    clearMarkBits<NumberHeap>();
+    clearMarkBits();
 
     // Mark stack roots.
@@ -1178 +1105 @@
     markStack.compact();
 
-    primaryHeap.operationInProgress = NoOperation;
-    numberHeap.operationInProgress = NoOperation;
+    m_heap.operationInProgress = NoOperation;
 }
 
 size_t Heap::objectCount() const
 {
-    return objectCount<PrimaryHeap>() + objectCount<NumberHeap>();
-}
-
-template <HeapType heapType> 
-size_t Heap::objectCount() const
-{
-    const CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-
-    return heap.nextBlock * HeapConstants<heapType>::cellsPerBlock // allocated full blocks
-           + heap.nextCell // allocated cells in current block
-           + markedCells<heapType>(heap.nextBlock, heap.nextCell) // marked cells in remainder of heap
-           - heap.usedBlocks; // 1 cell per block is a dummy sentinel
-}
-
-template <HeapType heapType> 
+    return m_heap.nextBlock * HeapConstants::cellsPerBlock // allocated full blocks
+           + m_heap.nextCell // allocated cells in current block
+           + markedCells(m_heap.nextBlock, m_heap.nextCell) // marked cells in remainder of m_heap
+           - m_heap.usedBlocks; // 1 cell per block is a dummy sentinel
+}
+
 void Heap::addToStatistics(Heap::Statistics& statistics) const
 {
-    const CollectorHeap& heap = heapType == PrimaryHeap ? primaryHeap : numberHeap;
-
-    statistics.size += heap.usedBlocks * BLOCK_SIZE;
-    statistics.free += heap.usedBlocks * BLOCK_SIZE - (objectCount<heapType>() * HeapConstants<heapType>::cellSize);
+    statistics.size += m_heap.usedBlocks * BLOCK_SIZE;
+    statistics.free += m_heap.usedBlocks * BLOCK_SIZE - (objectCount() * HeapConstants::cellSize);
 }
 
@@ -1210 +1125 @@
 {
     Statistics statistics = { 0, 0 };
-    addToStatistics<PrimaryHeap>(statistics);
-    addToStatistics<NumberHeap>(statistics);
+    addToStatistics(statistics);
     return statistics;
 }
@@ -1280 +1194 @@
 bool Heap::isBusy()
 {
-    return (primaryHeap.operationInProgress != NoOperation) | (numberHeap.operationInProgress != NoOperation);
+    return m_heap.operationInProgress != NoOperation;
 }
 
@@ -1291 +1205 @@
     JAVASCRIPTCORE_GC_MARKED();
 
-    primaryHeap.nextCell = 0;
-    primaryHeap.nextBlock = 0;
-    primaryHeap.extraCost = 0;
+    m_heap.nextCell = 0;
+    m_heap.nextBlock = 0;
+    m_heap.nextNumber = 0;
+    m_heap.extraCost = 0;
 #if ENABLE(JSC_ZOMBIES)
-    sweep<PrimaryHeap>();
-#endif
-    resizeBlocks<PrimaryHeap>();
-
-#if USE(JSVALUE32)
-    numberHeap.nextCell = 0;
-    numberHeap.nextBlock = 0;
-    resizeBlocks<NumberHeap>();
-#endif
+    sweep();
+#endif
+    resizeBlocks();
 
     JAVASCRIPTCORE_GC_END();
@@ -1315 +1224 @@
     // to clean up remaining garbage before marking. Otherwise, the conservative
     // marking mechanism might follow a pointer to unmapped memory.
-    if (primaryHeap.didShrink)
-        sweep<PrimaryHeap>();
+    if (m_heap.didShrink)
+        sweep();
 
     markRoots();
@@ -1322 +1231 @@
     JAVASCRIPTCORE_GC_MARKED();
 
-    primaryHeap.nextCell = 0;
-    primaryHeap.nextBlock = 0;
-    primaryHeap.extraCost = 0;
-    sweep<PrimaryHeap>();
-    resizeBlocks<PrimaryHeap>();
-
-#if USE(JSVALUE32)
-    numberHeap.nextCell = 0;
-    numberHeap.nextBlock = 0;
-    resizeBlocks<NumberHeap>();
-#endif
+    m_heap.nextCell = 0;
+    m_heap.nextBlock = 0;
+    m_heap.nextNumber = 0;
+    m_heap.extraCost = 0;
+    sweep();
+    resizeBlocks();
 
     JAVASCRIPTCORE_GC_END();
 }
 
-LiveObjectIterator<PrimaryHeap> Heap::primaryHeapBegin()
-{
-    return LiveObjectIterator<PrimaryHeap>(primaryHeap, 0);
-}
-
-LiveObjectIterator<PrimaryHeap> Heap::primaryHeapEnd()
-{
-    return LiveObjectIterator<PrimaryHeap>(primaryHeap, primaryHeap.usedBlocks);
+LiveObjectIterator Heap::primaryHeapBegin()
+{
+    return LiveObjectIterator(m_heap, 0);
+}
+
+LiveObjectIterator Heap::primaryHeapEnd()
+{
+    return LiveObjectIterator(m_heap, m_heap.usedBlocks);
 }
 

trunk/JavaScriptCore/runtime/Collector.h

@@ -48 +48 @@
 
     enum OperationInProgress { NoOperation, Allocation, Collection };
-    enum HeapType { PrimaryHeap, NumberHeap };
-
-    template <HeapType> class LiveObjectIterator;
+
+    class LiveObjectIterator;
 
     struct CollectorHeap {
         size_t nextBlock;
         size_t nextCell;
-
         CollectorBlock** blocks;
+        
+        void* nextNumber;
+
         size_t numBlocks;
         size_t usedBlocks;
 
         size_t extraCost;
-        
         bool didShrink;
 
@@ -114 +114 @@
         static bool isNumber(JSCell*);
         
-        LiveObjectIterator<PrimaryHeap> primaryHeapBegin();
-        LiveObjectIterator<PrimaryHeap> primaryHeapEnd();
+        LiveObjectIterator primaryHeapBegin();
+        LiveObjectIterator primaryHeapEnd();
 
     private:
-        template <HeapType heapType> void* heapAllocate(size_t);
         void reset();
-        void collectRemainingGarbage();
-        template <HeapType heapType> void sweep();
+        void sweep();
         static CollectorBlock* cellBlock(const JSCell*);
         static size_t cellOffset(const JSCell*);
@@ -129 +127 @@
         ~Heap();
 
-        template <HeapType heapType> NEVER_INLINE CollectorBlock* allocateBlock();
-        template <HeapType heapType> NEVER_INLINE void freeBlock(size_t);
-        NEVER_INLINE void freeBlock(CollectorBlock*);
-        template <HeapType heapType> void freeBlocks();
-        template <HeapType heapType> void resizeBlocks();
-        template <HeapType heapType> void growBlocks(size_t neededBlocks);
-        template <HeapType heapType> void shrinkBlocks(size_t neededBlocks);
-        template <HeapType heapType> void clearMarkBits();
-        template <HeapType heapType> void clearMarkBits(CollectorBlock*);
-        template <HeapType heapType> size_t markedCells(size_t startBlock = 0, size_t startCell = 0) const;
+        NEVER_INLINE CollectorBlock* allocateBlock();
+        NEVER_INLINE void freeBlock(size_t);
+        NEVER_INLINE void freeBlockPtr(CollectorBlock*);
+        void freeBlocks();
+        void resizeBlocks();
+        void growBlocks(size_t neededBlocks);
+        void shrinkBlocks(size_t neededBlocks);
+        void clearMarkBits();
+        void clearMarkBits(CollectorBlock*);
+        size_t markedCells(size_t startBlock = 0, size_t startCell = 0) const;
 
         void recordExtraCost(size_t);
 
-        template <HeapType heapType> void addToStatistics(Statistics&) const;
-        template <HeapType heapType> size_t objectCount() const;
+        void addToStatistics(Statistics&) const;
 
         void markRoots();
@@ -154 +151 @@
         typedef HashCountedSet<JSCell*> ProtectCountSet;
 
-        CollectorHeap primaryHeap;
-        CollectorHeap numberHeap;
+        CollectorHeap m_heap;
 
         ProtectCountSet m_protectedValues;
@@ -201 +197 @@
     const size_t CELL_MASK = CELL_SIZE - 1;
     const size_t CELL_ALIGN_MASK = ~CELL_MASK;
-    const size_t CELLS_PER_BLOCK = (BLOCK_SIZE - sizeof(Heap*) - sizeof(HeapType)) * 8 * CELL_SIZE / (8 * CELL_SIZE + 1) / CELL_SIZE; // one bitmap byte can represent 8 cells.
+    const size_t CELLS_PER_BLOCK = (BLOCK_SIZE - sizeof(Heap*)) * 8 * CELL_SIZE / (8 * CELL_SIZE + 1) / CELL_SIZE; // one bitmap byte can represent 8 cells.
     
-    const size_t SMALL_CELLS_PER_BLOCK = 2 * CELLS_PER_BLOCK;
     const size_t BITMAP_SIZE = (CELLS_PER_BLOCK + 7) / 8;
     const size_t BITMAP_WORDS = (BITMAP_SIZE + 3) / sizeof(uint32_t);
@@ -237 +232 @@
     };
 
-    struct SmallCollectorCell {
-        double memory[CELL_ARRAY_LENGTH / 2];
-    };
-
     class CollectorBlock {
     public:
@@ -246 +237 @@
         CollectorBitmap marked;
         Heap* heap;
-        HeapType type;
-    };
-
-    class SmallCellCollectorBlock {
-    public:
-        SmallCollectorCell cells[SMALL_CELLS_PER_BLOCK];
-        CollectorBitmap marked;
-        Heap* heap;
-        HeapType type;
-    };
-    
-    template <HeapType heapType> struct HeapConstants;
-
-    template <> struct HeapConstants<PrimaryHeap> {
+    };
+
+    struct HeapConstants {
         static const size_t cellSize = CELL_SIZE;
         static const size_t cellsPerBlock = CELLS_PER_BLOCK;
-        static const size_t bitmapShift = 0;
         typedef CollectorCell Cell;
         typedef CollectorBlock Block;
     };
 
-    template <> struct HeapConstants<NumberHeap> {
-        static const size_t cellSize = SMALL_CELL_SIZE;
-        static const size_t cellsPerBlock = SMALL_CELLS_PER_BLOCK;
-        static const size_t bitmapShift = 1;
-        typedef SmallCollectorCell Cell;
-        typedef SmallCellCollectorBlock Block;
-    };
-
     inline CollectorBlock* Heap::cellBlock(const JSCell* cell)
     {
         return reinterpret_cast<CollectorBlock*>(reinterpret_cast<uintptr_t>(cell) & BLOCK_MASK);
-    }
-
-    inline bool Heap::isNumber(JSCell* cell)
-    {
-        return Heap::cellBlock(cell)->type == NumberHeap;
     }
 
@@ -305 +271 @@
             recordExtraCost(cost);
     }
+    
+    inline void* Heap::allocateNumber(size_t s)
+    {
+        if (void* result = m_heap.nextNumber) {
+            m_heap.nextNumber = 0;
+            return result;
+        }
+
+        void* result = allocate(s);
+        m_heap.nextNumber = static_cast<char*>(result) + (CELL_SIZE / 2);
+        return result;
+    }
 
 } // namespace JSC

trunk/JavaScriptCore/runtime/CollectorHeapIterator.h

@@ -46 +46 @@
     };
 
-    template <HeapType heapType>
     class LiveObjectIterator : public CollectorHeapIterator {
     public:
         LiveObjectIterator(CollectorHeap&, size_t startBlock, size_t startCell = 0);
-        LiveObjectIterator<heapType>& operator++();
+        LiveObjectIterator& operator++();
     };
 
-    template <HeapType heapType>
     class DeadObjectIterator : public CollectorHeapIterator {
     public:
         DeadObjectIterator(CollectorHeap&, size_t startBlock, size_t startCell = 0);
-        DeadObjectIterator<heapType>& operator++();
+        DeadObjectIterator& operator++();
     };
 
-    template <HeapType heapType>
     class ObjectIterator : public CollectorHeapIterator {
     public:
         ObjectIterator(CollectorHeap&, size_t startBlock, size_t startCell = 0);
-        ObjectIterator<heapType>& operator++();
+        ObjectIterator& operator++();
     };
 
@@ -93 +90 @@
     }
 
-    template <HeapType heapType>
-    inline LiveObjectIterator<heapType>::LiveObjectIterator(CollectorHeap& heap, size_t startBlock, size_t startCell)
+    inline LiveObjectIterator::LiveObjectIterator(CollectorHeap& heap, size_t startBlock, size_t startCell)
         : CollectorHeapIterator(heap, startBlock, startCell - 1)
     {
@@ -100 +96 @@
     }
 
-    template <HeapType heapType>
-    inline LiveObjectIterator<heapType>& LiveObjectIterator<heapType>::operator++()
+    inline LiveObjectIterator& LiveObjectIterator::operator++()
     {
         if (m_block < m_heap.nextBlock || m_cell < m_heap.nextCell) {
-            advance(HeapConstants<heapType>::cellsPerBlock);
+            advance(HeapConstants::cellsPerBlock);
             return *this;
         }
 
         do {
-            advance(HeapConstants<heapType>::cellsPerBlock);
+            advance(HeapConstants::cellsPerBlock);
         } while (m_block < m_heap.usedBlocks && !m_heap.blocks[m_block]->marked.get(m_cell));
         return *this;
     }
 
-    template <HeapType heapType>
-    inline DeadObjectIterator<heapType>::DeadObjectIterator(CollectorHeap& heap, size_t startBlock, size_t startCell)
+    inline DeadObjectIterator::DeadObjectIterator(CollectorHeap& heap, size_t startBlock, size_t startCell)
         : CollectorHeapIterator(heap, startBlock, startCell - 1)
     {
@@ -121 +115 @@
     }
 
-    template <HeapType heapType>
-    inline DeadObjectIterator<heapType>& DeadObjectIterator<heapType>::operator++()
+    inline DeadObjectIterator& DeadObjectIterator::operator++()
     {
         do {
-            advance(HeapConstants<heapType>::cellsPerBlock);
+            advance(HeapConstants::cellsPerBlock);
             ASSERT(m_block > m_heap.nextBlock || (m_block == m_heap.nextBlock && m_cell >= m_heap.nextCell));
         } while (m_block < m_heap.usedBlocks && m_heap.blocks[m_block]->marked.get(m_cell));
@@ -131 +124 @@
     }
 
-    template <HeapType heapType>
-    inline ObjectIterator<heapType>::ObjectIterator(CollectorHeap& heap, size_t startBlock, size_t startCell)
+    inline ObjectIterator::ObjectIterator(CollectorHeap& heap, size_t startBlock, size_t startCell)
         : CollectorHeapIterator(heap, startBlock, startCell - 1)
     {
@@ -138 +130 @@
     }
 
-    template <HeapType heapType>
-    inline ObjectIterator<heapType>& ObjectIterator<heapType>::operator++()
+    inline ObjectIterator& ObjectIterator::operator++()
     {
-        advance(HeapConstants<heapType>::cellsPerBlock);
+        advance(HeapConstants::cellsPerBlock);
         return *this;
     }

trunk/JavaScriptCore/runtime/JSCell.h

@@ -134 +134 @@
     inline bool JSCell::isNumber() const
     {
-        return Heap::isNumber(const_cast<JSCell*>(this));
+        return m_structure->typeInfo().type() == NumberType;
     }
 #endif