Added large allocation support to MarkedSpace
https://bugs.webkit.org/show_bug.cgi?id=96214
Originally reviewed by Oliver Hunt, then I added a design revision by
suggested by Phil Pizlo.
I expanded the imprecise size classes to cover up to 32KB, then added
an mmap-based allocator for everything bigger. There's a lot of tuning
we could do in these size classes, but currently they're almost
completely unused, so I haven't done any tuning.
Subtle point: the large allocator is a degenerate case of our free list
logic. Its list only ever contains zero or one items.
* heap/Heap.h:
(JSC::Heap::allocateStructure): Pipe in size information.
* heap/MarkedAllocator.cpp:
(JSC::MarkedAllocator::tryAllocateHelper): Handle the case where we
find a free item in the sweep list but the item isn't big enough. This
can happen in the large allocator because it mixes sizes.
(JSC::MarkedAllocator::tryAllocate):
(JSC::MarkedAllocator::allocateSlowCase): More piping.
(JSC::MarkedAllocator::allocateBlock): Handle the oversize case.
(JSC::MarkedAllocator::addBlock): I moved the call to didAddBlock here
because it made more sense.
* heap/MarkedAllocator.h:
(MarkedAllocator):
(JSC::MarkedAllocator::allocate):
* heap/MarkedSpace.cpp:
(JSC::MarkedSpace::MarkedSpace):
(JSC::MarkedSpace::resetAllocators):
(JSC::MarkedSpace::canonicalizeCellLivenessData):
(JSC::MarkedSpace::isPagedOut):
(JSC::MarkedSpace::freeBlock):
* heap/MarkedSpace.h:
(MarkedSpace):
(JSC::MarkedSpace::allocatorFor):
(JSC::MarkedSpace::destructorAllocatorFor):
(JSC::MarkedSpace::allocateWithoutDestructor):
(JSC::MarkedSpace::allocateWithDestructor):
(JSC::MarkedSpace::allocateStructure):
(JSC::MarkedSpace::forEachBlock):
* runtime/Structure.h:
(JSC::Structure): More piping.
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@128141 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Source/JavaScriptCore/heap/MarkedAllocator.cpp b/Source/JavaScriptCore/heap/MarkedAllocator.cpp
index 20b5569..ab37ead 100644
--- a/Source/JavaScriptCore/heap/MarkedAllocator.cpp
+++ b/Source/JavaScriptCore/heap/MarkedAllocator.cpp
@@ -27,7 +27,7 @@
return false;
}
-inline void* MarkedAllocator::tryAllocateHelper()
+inline void* MarkedAllocator::tryAllocateHelper(size_t bytes)
{
if (!m_freeList.head) {
if (m_onlyContainsStructures && !m_heap->isSafeToSweepStructures()) {
@@ -42,12 +42,20 @@
}
for (MarkedBlock*& block = m_blocksToSweep; block; block = block->next()) {
- m_freeList = block->sweep(MarkedBlock::SweepToFreeList);
- if (m_freeList.head) {
- m_currentBlock = block;
- break;
+ MarkedBlock::FreeList freeList = block->sweep(MarkedBlock::SweepToFreeList);
+ if (!freeList.head) {
+ block->didConsumeFreeList();
+ continue;
}
- block->didConsumeFreeList();
+
+ if (bytes > block->cellSize()) {
+ block->zapFreeList(freeList);
+ continue;
+ }
+
+ m_currentBlock = block;
+ m_freeList = freeList;
+ break;
}
if (!m_freeList.head) {
@@ -62,16 +70,16 @@
return head;
}
-inline void* MarkedAllocator::tryAllocate()
+inline void* MarkedAllocator::tryAllocate(size_t bytes)
{
ASSERT(!m_heap->isBusy());
m_heap->m_operationInProgress = Allocation;
- void* result = tryAllocateHelper();
+ void* result = tryAllocateHelper(bytes);
m_heap->m_operationInProgress = NoOperation;
return result;
}
-void* MarkedAllocator::allocateSlowCase()
+void* MarkedAllocator::allocateSlowCase(size_t bytes)
{
ASSERT(m_heap->globalData()->apiLock().currentThreadIsHoldingLock());
#if COLLECT_ON_EVERY_ALLOCATION
@@ -82,7 +90,7 @@
ASSERT(!m_freeList.head);
m_heap->didAllocate(m_freeList.bytes);
- void* result = tryAllocate();
+ void* result = tryAllocate(bytes);
if (LIKELY(result != 0))
return result;
@@ -90,27 +98,39 @@
if (m_heap->shouldCollect()) {
m_heap->collect(Heap::DoNotSweep);
- result = tryAllocate();
+ result = tryAllocate(bytes);
if (result)
return result;
}
ASSERT(!m_heap->shouldCollect());
- MarkedBlock* block = allocateBlock();
+ MarkedBlock* block = allocateBlock(bytes);
ASSERT(block);
addBlock(block);
- result = tryAllocate();
+ result = tryAllocate(bytes);
ASSERT(result);
return result;
}
-MarkedBlock* MarkedAllocator::allocateBlock()
+MarkedBlock* MarkedAllocator::allocateBlock(size_t bytes)
{
- MarkedBlock* block = MarkedBlock::create(m_heap->blockAllocator().allocate(), m_heap, m_cellSize, m_cellsNeedDestruction, m_onlyContainsStructures);
- m_markedSpace->didAddBlock(block);
- return block;
+ size_t minBlockSize = MarkedBlock::blockSize;
+ size_t minAllocationSize = WTF::roundUpToMultipleOf(WTF::pageSize(), sizeof(MarkedBlock) + bytes);
+ size_t blockSize = std::max(minBlockSize, minAllocationSize);
+
+ size_t cellSize = m_cellSize ? m_cellSize : WTF::roundUpToMultipleOf<MarkedBlock::atomSize>(bytes);
+
+ if (blockSize == MarkedBlock::blockSize) {
+ PageAllocationAligned allocation = m_heap->blockAllocator().allocate();
+ return MarkedBlock::create(allocation, m_heap, cellSize, m_cellsNeedDestruction, m_onlyContainsStructures);
+ }
+
+ PageAllocationAligned allocation = PageAllocationAligned::allocate(blockSize, MarkedBlock::blockSize, OSAllocator::JSGCHeapPages);
+ if (!static_cast<bool>(allocation))
+ CRASH();
+ return MarkedBlock::create(allocation, m_heap, cellSize, m_cellsNeedDestruction, m_onlyContainsStructures);
}
void MarkedAllocator::addBlock(MarkedBlock* block)
@@ -121,6 +141,7 @@
m_blockList.append(block);
m_blocksToSweep = m_currentBlock = block;
m_freeList = block->sweep(MarkedBlock::SweepToFreeList);
+ m_markedSpace->didAddBlock(block);
}
void MarkedAllocator::removeBlock(MarkedBlock* block)
